feat: add AI catalog description generation
This commit is contained in:
Codex
+51
-2
@@ -2,7 +2,7 @@ import Fastify, { type FastifyInstance, type FastifyRequest } from "fastify";
|
||||
import cors from "@fastify/cors";
|
||||
import cookie from "@fastify/cookie";
|
||||
import rateLimit from "@fastify/rate-limit";
|
||||
import { join } from "node:path";
|
||||
import { dirname, isAbsolute, join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import type { AppConfig } from "./config.js";
|
||||
import { ThtRunner } from "./tht/tht-runner.js";
|
||||
@@ -48,6 +48,19 @@ import { catalogTableRoutes } from "./routes/catalog-tables.js";
|
||||
import { ConcreteCatalogSchemaIntrospector, type CatalogSchemaIntrospector } from "./catalog/schema-introspector.js";
|
||||
import { CatalogSyncWorker } from "./catalog/sync-worker.js";
|
||||
import { catalogSchemaRoutes } from "./routes/catalog-schema.js";
|
||||
import {
|
||||
loadMetadataGenerationModels,
|
||||
type MetadataGenerationModels,
|
||||
} from "./catalog/metadata-generation-models.js";
|
||||
import { metadataGenerationModelRoutes } from "./routes/metadata-generation-models.js";
|
||||
import { catalogDescriptionConsolidationRoutes } from "./routes/catalog-description-consolidation.js";
|
||||
import { PythonModelCompleter, type ModelCompleter } from "./catalog/model-completer.js";
|
||||
import { DescriptionGenerationWorker } from "./catalog/description-generation-worker.js";
|
||||
import {
|
||||
PostgresDescriptionSourceSampler,
|
||||
type DescriptionSourceSampler,
|
||||
} from "./catalog/description-source-sampler.js";
|
||||
import { catalogDescriptionGenerationRoutes } from "./routes/catalog-description-generation.js";
|
||||
|
||||
export interface BuildAppDeps {
|
||||
thtRunner?: ThtRunner;
|
||||
@@ -67,6 +80,9 @@ export interface BuildAppDeps {
|
||||
catalogSchemaIntrospector?: CatalogSchemaIntrospector;
|
||||
catalogSyncWorker?: CatalogSyncWorker;
|
||||
catalogOperationCoordinator?: CatalogOperationCoordinator;
|
||||
metadataGenerationModels?: MetadataGenerationModels;
|
||||
modelCompleter?: ModelCompleter;
|
||||
descriptionSourceSampler?: DescriptionSourceSampler;
|
||||
workspaceRuntimeSupport?: (workspace: WorkspaceDescriptor) => boolean;
|
||||
maintenanceBarrier?: MaintenanceBarrier;
|
||||
piManagement?: PiManagementService;
|
||||
@@ -141,10 +157,28 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
|
||||
const workspaceRegistry = deps?.workspaceRegistry ?? new WorkspaceRegistry(config.workspaceRegistry);
|
||||
const catalogRepository = deps?.catalogRepository ?? createCatalogRepository(config.catalogDatabase);
|
||||
const catalogOperationCoordinator = deps?.catalogOperationCoordinator ?? new CatalogOperationCoordinator();
|
||||
const metadataGenerationModels = deps?.metadataGenerationModels ?? loadMetadataGenerationModels({
|
||||
installationFile: config.installationConfigFile,
|
||||
secretsFile: config.secretsFile,
|
||||
});
|
||||
const modelCompleter = deps?.modelCompleter ?? new PythonModelCompleter({
|
||||
pythonExecutable: isAbsolute(config.thtBin) ? join(dirname(config.thtBin), "python") : "python3",
|
||||
cwd: config.harnessDir,
|
||||
});
|
||||
const catalogPostgresAccess = deps?.catalogPostgresAccess ?? new ConcreteCatalogPostgresAccess(
|
||||
workspaceSecretStore,
|
||||
{ connectTimeoutMs: config.workspaceDiagnosticTimeoutMs },
|
||||
);
|
||||
const descriptionSourceSampler = deps?.descriptionSourceSampler
|
||||
?? new PostgresDescriptionSourceSampler(catalogPostgresAccess);
|
||||
const descriptionGenerationWorker = new DescriptionGenerationWorker(
|
||||
catalogRepository,
|
||||
workspaceRegistry,
|
||||
metadataGenerationModels,
|
||||
modelCompleter,
|
||||
catalogOperationCoordinator,
|
||||
descriptionSourceSampler,
|
||||
);
|
||||
const catalogService = deps?.catalogService ?? new CatalogService(
|
||||
catalogRepository,
|
||||
workspaceRegistry,
|
||||
@@ -166,10 +200,12 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
|
||||
config.catalogSyncTimeoutMs,
|
||||
);
|
||||
app.addHook("onReady", async () => { await catalogSyncWorker.initialize(); });
|
||||
app.addHook("onReady", async () => { await descriptionGenerationWorker.initialize(); });
|
||||
if (!deps?.catalogRepository && catalogRepository.close) {
|
||||
app.addHook("onClose", async () => { await catalogRepository.close?.(); });
|
||||
}
|
||||
app.addHook("onClose", async () => { await catalogSyncWorker.stop(); });
|
||||
app.addHook("onClose", async () => { await descriptionGenerationWorker.stop(); });
|
||||
const workspaceDiagnoser = deps?.workspaceDiagnoser
|
||||
?? createProductionWorkspaceDiagnoser(config.workspaceDiagnosticTimeoutMs, undefined, {
|
||||
internalQdrantUrl: config.internalQdrantUrl,
|
||||
@@ -384,12 +420,25 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
|
||||
secretStore: workspaceSecretStore,
|
||||
});
|
||||
catalogDatabaseRoutes(app, { repository: catalogRepository, service: catalogService, operations: catalogOperationCoordinator });
|
||||
catalogTableRoutes(app, { repository: catalogRepository, service: catalogTableService });
|
||||
catalogTableRoutes(app, {
|
||||
repository: catalogRepository,
|
||||
service: catalogTableService,
|
||||
operations: catalogOperationCoordinator,
|
||||
});
|
||||
catalogSchemaRoutes(app, {
|
||||
repository: catalogRepository,
|
||||
worker: catalogSyncWorker,
|
||||
operations: catalogOperationCoordinator,
|
||||
});
|
||||
catalogDescriptionConsolidationRoutes(app, {
|
||||
repository: catalogRepository,
|
||||
operations: catalogOperationCoordinator,
|
||||
});
|
||||
metadataGenerationModelRoutes(app, metadataGenerationModels);
|
||||
catalogDescriptionGenerationRoutes(app, {
|
||||
repository: catalogRepository,
|
||||
worker: descriptionGenerationWorker,
|
||||
});
|
||||
settingsRoutes(app, { cfg: config, listModels, getSettings });
|
||||
piManagementRoutes(app, { service: piManagement });
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,163 @@
|
||||
import type { CatalogPostgresAccess } from "./postgres-access.js";
|
||||
import type { WorkspaceDatabase } from "./types.js";
|
||||
|
||||
const MAX_SOURCE_ROWS = 5;
|
||||
const MAX_REPRESENTATIVE_VALUES = 5;
|
||||
const MAX_SOURCE_COLUMNS_PER_TARGET = 8;
|
||||
const MAX_SOURCE_VALUE_BYTES = 256;
|
||||
|
||||
export type DescriptionSourceSampleValue = string | number | boolean | null;
|
||||
|
||||
export interface DescriptionSourceSampleField {
|
||||
name: string;
|
||||
value: DescriptionSourceSampleValue;
|
||||
}
|
||||
|
||||
export interface DescriptionSourceSampleRow {
|
||||
fields: readonly DescriptionSourceSampleField[];
|
||||
}
|
||||
|
||||
export interface DescriptionSourceRepresentativeValues {
|
||||
column: string;
|
||||
values: readonly Exclude<DescriptionSourceSampleValue, null>[];
|
||||
}
|
||||
|
||||
export interface DescriptionTargetSourceSample {
|
||||
targetId: string;
|
||||
tableName: string;
|
||||
rows: readonly DescriptionSourceSampleRow[];
|
||||
representativeValues: readonly DescriptionSourceRepresentativeValues[];
|
||||
}
|
||||
|
||||
export interface DescriptionSourceSamplingTarget {
|
||||
targetId: string;
|
||||
tableName: string;
|
||||
columnNames: readonly string[];
|
||||
}
|
||||
|
||||
/** Optional, transient source context for one model-completion batch. */
|
||||
export interface DescriptionSourceSampler {
|
||||
sample(
|
||||
database: WorkspaceDatabase,
|
||||
targets: readonly DescriptionSourceSamplingTarget[],
|
||||
signal: AbortSignal,
|
||||
): Promise<readonly DescriptionTargetSourceSample[]>;
|
||||
}
|
||||
|
||||
function quoteIdentifier(identifier: string): string {
|
||||
return `"${identifier.replaceAll('"', '""')}"`;
|
||||
}
|
||||
|
||||
function boundedUtf8(value: string, maxBytes: number): string {
|
||||
const normalized = value
|
||||
.normalize("NFC")
|
||||
.replace(/\r\n?/g, "\n")
|
||||
.replace(/[\u0000-\u0008\u000b\u000c\u000e-\u001f\u007f]/g, " ");
|
||||
if (Buffer.byteLength(normalized, "utf8") <= maxBytes) return normalized;
|
||||
let result = "";
|
||||
let bytes = 0;
|
||||
for (const character of normalized) {
|
||||
const characterBytes = Buffer.byteLength(character, "utf8");
|
||||
if (bytes + characterBytes > maxBytes) break;
|
||||
result += character;
|
||||
bytes += characterBytes;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function normalizeValue(value: unknown): DescriptionSourceSampleValue | undefined {
|
||||
if (value === null) return null;
|
||||
if (typeof value === "string") return boundedUtf8(value, MAX_SOURCE_VALUE_BYTES);
|
||||
if (typeof value === "boolean") return value;
|
||||
if (typeof value === "number") return Number.isFinite(value) ? value : undefined;
|
||||
if (typeof value === "bigint") return boundedUtf8(String(value), MAX_SOURCE_VALUE_BYTES);
|
||||
if (value instanceof Date && !Number.isNaN(value.valueOf())) return value.toISOString();
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function distinctKey(value: Exclude<DescriptionSourceSampleValue, null>): string {
|
||||
return `${typeof value}:${String(value)}`;
|
||||
}
|
||||
|
||||
/** PostgreSQL-wire sampler. REST bindings remain unsupported by CatalogPostgresAccess. */
|
||||
export class PostgresDescriptionSourceSampler implements DescriptionSourceSampler {
|
||||
constructor(private readonly access: CatalogPostgresAccess) {}
|
||||
|
||||
async sample(
|
||||
database: WorkspaceDatabase,
|
||||
targets: readonly DescriptionSourceSamplingTarget[],
|
||||
signal: AbortSignal,
|
||||
): Promise<readonly DescriptionTargetSourceSample[]> {
|
||||
const client = await this.access.connect(database, signal);
|
||||
let transactionOpen = false;
|
||||
try {
|
||||
await client.query("BEGIN TRANSACTION READ ONLY", []);
|
||||
transactionOpen = true;
|
||||
const samples: DescriptionTargetSourceSample[] = [];
|
||||
for (const target of targets) {
|
||||
const columnNames = [...new Set(target.columnNames)].slice(0, MAX_SOURCE_COLUMNS_PER_TARGET);
|
||||
if (columnNames.length === 0) {
|
||||
samples.push({
|
||||
targetId: target.targetId,
|
||||
tableName: target.tableName,
|
||||
rows: [],
|
||||
representativeValues: [],
|
||||
});
|
||||
continue;
|
||||
}
|
||||
const projections = columnNames.map((columnName) => {
|
||||
const identifier = quoteIdentifier(columnName);
|
||||
return `LEFT((${identifier})::text, $1) AS ${identifier}`;
|
||||
});
|
||||
const sql = [
|
||||
`SELECT ${projections.join(", ")}`,
|
||||
`FROM ${quoteIdentifier(database.schema)}.${quoteIdentifier(target.tableName)}`,
|
||||
"LIMIT $2",
|
||||
].join(" ");
|
||||
const result = await client.query(sql, [MAX_SOURCE_VALUE_BYTES, MAX_SOURCE_ROWS]);
|
||||
const rows = result.rows.slice(0, MAX_SOURCE_ROWS).map((row) => ({
|
||||
fields: columnNames.flatMap((name) => {
|
||||
const value = normalizeValue(row[name]);
|
||||
return value === undefined ? [] : [{ name, value }];
|
||||
}),
|
||||
}));
|
||||
const valuesByColumn = new Map<
|
||||
string,
|
||||
Exclude<DescriptionSourceSampleValue, null>[]
|
||||
>();
|
||||
const seenByColumn = new Map<string, Set<string>>();
|
||||
let representativeValueCount = 0;
|
||||
for (const row of rows) {
|
||||
for (const field of row.fields) {
|
||||
if (representativeValueCount === MAX_REPRESENTATIVE_VALUES) break;
|
||||
if (field.value === null) continue;
|
||||
const seen = seenByColumn.get(field.name) ?? new Set<string>();
|
||||
const key = distinctKey(field.value);
|
||||
if (seen.has(key)) continue;
|
||||
seen.add(key);
|
||||
seenByColumn.set(field.name, seen);
|
||||
const values = valuesByColumn.get(field.name) ?? [];
|
||||
values.push(field.value);
|
||||
valuesByColumn.set(field.name, values);
|
||||
representativeValueCount += 1;
|
||||
}
|
||||
if (representativeValueCount === MAX_REPRESENTATIVE_VALUES) break;
|
||||
}
|
||||
const representativeValues = columnNames.flatMap((column) => {
|
||||
const values = valuesByColumn.get(column);
|
||||
return values && values.length > 0 ? [{ column, values }] : [];
|
||||
});
|
||||
samples.push({
|
||||
targetId: target.targetId,
|
||||
tableName: target.tableName,
|
||||
rows,
|
||||
representativeValues,
|
||||
});
|
||||
}
|
||||
return samples;
|
||||
} finally {
|
||||
if (transactionOpen) await client.query("ROLLBACK", []).catch(() => undefined);
|
||||
await client.end().catch(() => undefined);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2,7 +2,10 @@ import { randomUUID } from "node:crypto";
|
||||
import {
|
||||
CatalogConflictError,
|
||||
CatalogConnectorError,
|
||||
DescriptionGenerationRunActiveError,
|
||||
type CatalogColumn,
|
||||
type CatalogDescriptionConsolidationCounts,
|
||||
type CatalogDescriptionTarget,
|
||||
type CatalogDatabaseMetadataDeleteTarget,
|
||||
type CatalogMetadataDeleteCounts,
|
||||
type CatalogRelationship,
|
||||
@@ -19,6 +22,10 @@ import {
|
||||
type DatabaseTestResult,
|
||||
type ObservedCatalogTable,
|
||||
type ObservedSchemaSnapshot,
|
||||
type DescriptionGenerationEvent,
|
||||
type DescriptionGenerationRun,
|
||||
type DescriptionGenerationRunUpdate,
|
||||
type DescriptionGenerationScope,
|
||||
type TableSyncRepositoryResult,
|
||||
type WorkspaceDatabase,
|
||||
} from "./types.js";
|
||||
@@ -33,6 +40,8 @@ export class MemoryCatalogRepository implements CatalogRepository {
|
||||
private readonly tables = new Map<string, CatalogTable>();
|
||||
private readonly columns = new Map<string, CatalogColumn>();
|
||||
private readonly relationships = new Map<string, CatalogRelationship>();
|
||||
private readonly descriptionGenerationRuns = new Map<string, DescriptionGenerationRun>();
|
||||
private readonly descriptionGenerationEvents = new Map<string, DescriptionGenerationEvent[]>();
|
||||
private readonly syncRuns = new Map<string, CatalogSyncRun>();
|
||||
private readonly syncEvents = new Map<string, CatalogSyncEvent[]>();
|
||||
|
||||
@@ -122,6 +131,11 @@ export class MemoryCatalogRepository implements CatalogRepository {
|
||||
for (const [relationshipId, relationship] of this.relationships) {
|
||||
if (relationship.databaseId === id) this.relationships.delete(relationshipId);
|
||||
}
|
||||
for (const [runId, run] of this.descriptionGenerationRuns) {
|
||||
if (run.databaseId !== id) continue;
|
||||
this.descriptionGenerationRuns.delete(runId);
|
||||
this.descriptionGenerationEvents.delete(runId);
|
||||
}
|
||||
return this.records.delete(id);
|
||||
}
|
||||
async listTables(databaseId: string): Promise<CatalogTable[]> {
|
||||
@@ -196,6 +210,166 @@ export class MemoryCatalogRepository implements CatalogRepository {
|
||||
return structuredClone(updated);
|
||||
}
|
||||
|
||||
async consolidateGeneratedDescriptions(
|
||||
databaseId: string,
|
||||
target: CatalogDescriptionTarget,
|
||||
targetIds: readonly string[],
|
||||
): Promise<CatalogDescriptionConsolidationCounts | undefined> {
|
||||
const selectedTargetIds = [...new Set(targetIds)];
|
||||
if (!this.records.has(databaseId) || selectedTargetIds.length === 0) {
|
||||
return undefined;
|
||||
}
|
||||
const now = new Date().toISOString();
|
||||
if (target === "tables") {
|
||||
const targets = selectedTargetIds.map((id) => this.tables.get(id));
|
||||
if (targets.some((table) => !table || table.databaseId !== databaseId)) return undefined;
|
||||
const copied = targets.filter((table) => Boolean(table!.generatedDescription?.trim()));
|
||||
for (const table of copied) {
|
||||
this.tables.set(table!.id, {
|
||||
...table!,
|
||||
description: table!.generatedDescription,
|
||||
version: table!.version + 1,
|
||||
updatedAt: now,
|
||||
});
|
||||
}
|
||||
return { copied: copied.length, skipped: targets.length - copied.length };
|
||||
}
|
||||
|
||||
const targets = selectedTargetIds.map((id) => this.columns.get(id));
|
||||
if (targets.some((column) => (
|
||||
!column || this.tables.get(column.tableId)?.databaseId !== databaseId
|
||||
))) return undefined;
|
||||
const copied = targets.filter((column) => Boolean(column!.generatedDescription?.trim()));
|
||||
for (const column of copied) {
|
||||
this.columns.set(column!.id, {
|
||||
...column!,
|
||||
description: column!.generatedDescription,
|
||||
version: column!.version + 1,
|
||||
updatedAt: now,
|
||||
});
|
||||
}
|
||||
return { copied: copied.length, skipped: targets.length - copied.length };
|
||||
}
|
||||
|
||||
async createDescriptionGenerationRun(
|
||||
databaseId: string,
|
||||
scope: DescriptionGenerationScope,
|
||||
modelId: string,
|
||||
language: DescriptionGenerationRun["language"],
|
||||
total: number,
|
||||
): Promise<DescriptionGenerationRun> {
|
||||
if ([...this.descriptionGenerationRuns.values()].some((run) => (
|
||||
run.status === "queued" || run.status === "running"
|
||||
))) {
|
||||
throw new DescriptionGenerationRunActiveError("A description generation run is already active");
|
||||
}
|
||||
const now = new Date().toISOString();
|
||||
const run: DescriptionGenerationRun = {
|
||||
id: randomUUID(),
|
||||
databaseId,
|
||||
scope,
|
||||
modelId,
|
||||
language,
|
||||
status: "queued",
|
||||
total,
|
||||
processed: 0,
|
||||
generated: 0,
|
||||
nonGeneratable: 0,
|
||||
failed: 0,
|
||||
createdAt: now,
|
||||
startedAt: null,
|
||||
updatedAt: now,
|
||||
finishedAt: null,
|
||||
errorSummary: null,
|
||||
};
|
||||
this.descriptionGenerationRuns.set(run.id, run);
|
||||
return structuredClone(run);
|
||||
}
|
||||
|
||||
async getDescriptionGenerationRun(runId: string): Promise<DescriptionGenerationRun | undefined> {
|
||||
const run = this.descriptionGenerationRuns.get(runId);
|
||||
return run ? structuredClone(run) : undefined;
|
||||
}
|
||||
|
||||
async listDescriptionGenerationRuns(limit = 50): Promise<DescriptionGenerationRun[]> {
|
||||
return [...this.descriptionGenerationRuns.values()]
|
||||
.sort((a, b) => b.createdAt.localeCompare(a.createdAt) || b.id.localeCompare(a.id))
|
||||
.slice(0, limit)
|
||||
.map((run) => structuredClone(run));
|
||||
}
|
||||
|
||||
async getActiveDescriptionGenerationRun(): Promise<DescriptionGenerationRun | undefined> {
|
||||
const run = [...this.descriptionGenerationRuns.values()]
|
||||
.filter((candidate) => candidate.status === "queued" || candidate.status === "running")
|
||||
.sort((a, b) => b.createdAt.localeCompare(a.createdAt))[0];
|
||||
return run ? structuredClone(run) : undefined;
|
||||
}
|
||||
|
||||
async interruptActiveDescriptionGenerationRuns(
|
||||
errorSummary: string,
|
||||
): Promise<DescriptionGenerationRun[]> {
|
||||
const interrupted: DescriptionGenerationRun[] = [];
|
||||
for (const run of this.descriptionGenerationRuns.values()) {
|
||||
if (run.status !== "queued" && run.status !== "running") continue;
|
||||
const now = new Date().toISOString();
|
||||
const updated: DescriptionGenerationRun = {
|
||||
...run,
|
||||
status: "interrupted",
|
||||
updatedAt: now,
|
||||
finishedAt: now,
|
||||
errorSummary,
|
||||
};
|
||||
this.descriptionGenerationRuns.set(run.id, updated);
|
||||
interrupted.push(structuredClone(updated));
|
||||
}
|
||||
return interrupted;
|
||||
}
|
||||
|
||||
async updateDescriptionGenerationRun(
|
||||
runId: string,
|
||||
update: DescriptionGenerationRunUpdate,
|
||||
): Promise<DescriptionGenerationRun | undefined> {
|
||||
const current = this.descriptionGenerationRuns.get(runId);
|
||||
if (!current) return undefined;
|
||||
const updated = {
|
||||
...current,
|
||||
...structuredClone(update),
|
||||
updatedAt: new Date().toISOString(),
|
||||
};
|
||||
this.descriptionGenerationRuns.set(runId, updated);
|
||||
return structuredClone(updated);
|
||||
}
|
||||
|
||||
async appendDescriptionGenerationEvent(
|
||||
runId: string,
|
||||
level: DescriptionGenerationEvent["level"],
|
||||
message: string,
|
||||
): Promise<DescriptionGenerationEvent> {
|
||||
if (!this.descriptionGenerationRuns.has(runId)) {
|
||||
throw new CatalogConflictError("Description Generation Run does not exist");
|
||||
}
|
||||
const events = this.descriptionGenerationEvents.get(runId) ?? [];
|
||||
const event: DescriptionGenerationEvent = {
|
||||
runId,
|
||||
sequence: events.length + 1,
|
||||
level,
|
||||
message,
|
||||
createdAt: new Date().toISOString(),
|
||||
};
|
||||
events.push(event);
|
||||
this.descriptionGenerationEvents.set(runId, events);
|
||||
return structuredClone(event);
|
||||
}
|
||||
|
||||
async listDescriptionGenerationEvents(
|
||||
runId: string,
|
||||
afterSequence = 0,
|
||||
): Promise<DescriptionGenerationEvent[]> {
|
||||
return (this.descriptionGenerationEvents.get(runId) ?? [])
|
||||
.filter((event) => event.sequence > afterSequence)
|
||||
.map((event) => structuredClone(event));
|
||||
}
|
||||
|
||||
async listRelationships(databaseId: string): Promise<CatalogRelationship[]> {
|
||||
return [...this.relationships.values()].filter((relationship) => relationship.databaseId === databaseId)
|
||||
.sort((a, b) => `${a.sourceTableName}.${a.constraintName}`.localeCompare(`${b.sourceTableName}.${b.constraintName}`))
|
||||
|
||||
@@ -0,0 +1,235 @@
|
||||
import {
|
||||
closeSync, constants, fstatSync, lstatSync, openSync, readFileSync,
|
||||
type Stats,
|
||||
} from "node:fs";
|
||||
import { parseAllDocuments } from "yaml";
|
||||
import { z } from "zod";
|
||||
import {
|
||||
loadSecretBundle,
|
||||
METADATA_GENERATION_SECRET_KEYS,
|
||||
} from "../config/secret-bundle.js";
|
||||
|
||||
const MAX_INSTALLATION_BYTES = 1024 * 1024;
|
||||
const RUNTIME_INSTALLATION_FILE = "/run/thothii-installation/thothii-installation.yaml";
|
||||
const modelId = z.string().regex(/^[a-z][a-z0-9._-]{0,63}$/);
|
||||
const apiKeyEnvironment = z.enum(METADATA_GENERATION_SECRET_KEYS);
|
||||
const endpointSchema = z.object({
|
||||
baseUrl: z.string().min(1).max(2048).refine((value) => {
|
||||
try {
|
||||
const url = new URL(value);
|
||||
return (url.protocol === "http:" || url.protocol === "https:")
|
||||
&& url.username === "" && url.password === "" && url.search === "" && url.hash === "";
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}),
|
||||
apiVersion: z.string().regex(/^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/).optional(),
|
||||
}).strict();
|
||||
const configuredModelSchema = z.object({
|
||||
id: modelId,
|
||||
label: z.string().min(1).max(128).refine((value) => value.trim() === value && !/\p{Cc}/u.test(value)),
|
||||
litellm: z.object({
|
||||
provider: z.string().regex(/^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/),
|
||||
model: z.string().regex(/^[A-Za-z0-9][A-Za-z0-9._:/-]{0,255}$/),
|
||||
disableThinking: z.literal(true).optional(),
|
||||
endpoint: endpointSchema.optional(),
|
||||
}).strict(),
|
||||
apiKeyEnv: apiKeyEnvironment.optional(),
|
||||
}).strict().superRefine((value, context) => {
|
||||
if (value.apiKeyEnv === undefined && value.litellm.endpoint === undefined) {
|
||||
context.addIssue({
|
||||
code: z.ZodIssueCode.custom,
|
||||
path: ["apiKeyEnv"],
|
||||
message: "keyless models require an explicit endpoint",
|
||||
});
|
||||
}
|
||||
if (value.litellm.disableThinking === true && value.litellm.endpoint === undefined) {
|
||||
context.addIssue({
|
||||
code: z.ZodIssueCode.custom,
|
||||
path: ["litellm", "disableThinking"],
|
||||
message: "thinking may be disabled only for an explicit endpoint",
|
||||
});
|
||||
}
|
||||
});
|
||||
const metadataGenerationSchema = z.object({
|
||||
default: modelId.optional(),
|
||||
models: z.array(configuredModelSchema).max(64).default([]),
|
||||
}).strict();
|
||||
const installationSchema = z.object({
|
||||
metadataGeneration: metadataGenerationSchema.optional(),
|
||||
}).passthrough();
|
||||
|
||||
export interface MetadataGenerationModelChoice {
|
||||
id: string;
|
||||
label: string;
|
||||
}
|
||||
|
||||
export interface MetadataGenerationModelCatalog {
|
||||
models: MetadataGenerationModelChoice[];
|
||||
default: string | null;
|
||||
}
|
||||
|
||||
export interface ResolvedMetadataGenerationModel {
|
||||
readonly id: string;
|
||||
readonly provider: string;
|
||||
readonly model: string;
|
||||
readonly disableThinking?: true;
|
||||
readonly endpoint?: Readonly<{ baseUrl: string; apiVersion?: string }>;
|
||||
readonly apiKeyEnv?: string;
|
||||
readonly apiKey?: string;
|
||||
}
|
||||
|
||||
export class MetadataGenerationModelUnavailableError extends Error {
|
||||
constructor() {
|
||||
super("metadata-generation model is unavailable");
|
||||
this.name = "MetadataGenerationModelUnavailableError";
|
||||
}
|
||||
}
|
||||
|
||||
/** The complete interface callers need: safe discovery plus fail-closed runtime resolution. */
|
||||
export interface MetadataGenerationModels {
|
||||
catalog(): MetadataGenerationModelCatalog;
|
||||
resolve(selection: string): ResolvedMetadataGenerationModel;
|
||||
}
|
||||
|
||||
class RestartLoadedMetadataGenerationModels implements MetadataGenerationModels {
|
||||
readonly #models: ReadonlyMap<string, ResolvedMetadataGenerationModel>;
|
||||
readonly #catalog: MetadataGenerationModelCatalog;
|
||||
|
||||
constructor(
|
||||
models: ReadonlyMap<string, ResolvedMetadataGenerationModel> = new Map(),
|
||||
defaultModel: string | null = null,
|
||||
choices: MetadataGenerationModelChoice[] = [],
|
||||
) {
|
||||
this.#models = models;
|
||||
this.#catalog = {
|
||||
models: choices.map((choice) => ({ ...choice })),
|
||||
default: defaultModel,
|
||||
};
|
||||
}
|
||||
|
||||
catalog(): MetadataGenerationModelCatalog {
|
||||
return {
|
||||
models: this.#catalog.models.map((choice) => ({ ...choice })),
|
||||
default: this.#catalog.default,
|
||||
};
|
||||
}
|
||||
|
||||
resolve(selection: string): ResolvedMetadataGenerationModel {
|
||||
const model = typeof selection === "string" ? this.#models.get(selection) : undefined;
|
||||
if (!model) throw new MetadataGenerationModelUnavailableError();
|
||||
return model;
|
||||
}
|
||||
}
|
||||
|
||||
function invalid(message = "metadata-generation configuration is invalid"): Error {
|
||||
return new Error(message);
|
||||
}
|
||||
|
||||
function protectedInstallationStat(file: string, info: Stats): boolean {
|
||||
const mode = info.mode & 0o777;
|
||||
if (!info.isFile() || info.isSymbolicLink() || info.nlink !== 1
|
||||
|| info.size < 1 || info.size > MAX_INSTALLATION_BYTES) return false;
|
||||
if (file === RUNTIME_INSTALLATION_FILE && info.uid === 0 && mode === 0o444) return true;
|
||||
return info.uid === (process.getuid?.() ?? info.uid) && (mode === 0o400 || mode === 0o600);
|
||||
}
|
||||
|
||||
function readProtectedInstallation(file: string): string {
|
||||
let descriptor: number | undefined;
|
||||
try {
|
||||
const before = lstatSync(file);
|
||||
if (!protectedInstallationStat(file, before)) throw new Error("unavailable");
|
||||
descriptor = openSync(file, constants.O_RDONLY | constants.O_NOFOLLOW);
|
||||
const opened = fstatSync(descriptor);
|
||||
if (!protectedInstallationStat(file, opened)
|
||||
|| before.dev !== opened.dev || before.ino !== opened.ino) throw new Error("unavailable");
|
||||
const source = readFileSync(descriptor, "utf8");
|
||||
const after = fstatSync(descriptor);
|
||||
const current = lstatSync(file);
|
||||
if (!protectedInstallationStat(file, after) || !protectedInstallationStat(file, current)
|
||||
|| opened.dev !== after.dev || opened.ino !== after.ino
|
||||
|| opened.dev !== current.dev || opened.ino !== current.ino) throw new Error("unavailable");
|
||||
return source;
|
||||
} finally {
|
||||
if (descriptor !== undefined) try { closeSync(descriptor); } catch { /* sanitized below */ }
|
||||
}
|
||||
}
|
||||
|
||||
function readInstallation(file: string): unknown {
|
||||
try {
|
||||
const documents = parseAllDocuments(readProtectedInstallation(file), { uniqueKeys: true });
|
||||
if (documents.length !== 1) throw invalid("metadata-generation installation must contain one YAML document");
|
||||
const document = documents[0];
|
||||
if (document.errors.length > 0 || document.warnings.length > 0) {
|
||||
throw invalid("metadata-generation installation contains invalid YAML");
|
||||
}
|
||||
return document.toJSON();
|
||||
} catch (error) {
|
||||
if (error instanceof Error && error.message.startsWith("metadata-generation")) throw error;
|
||||
throw invalid("metadata-generation installation is unavailable");
|
||||
}
|
||||
}
|
||||
|
||||
export function loadMetadataGenerationModels(options: {
|
||||
installationFile?: string;
|
||||
secretsFile?: string;
|
||||
}): MetadataGenerationModels {
|
||||
if (!options.installationFile) return new RestartLoadedMetadataGenerationModels();
|
||||
const installation = installationSchema.safeParse(readInstallation(options.installationFile));
|
||||
if (!installation.success) throw invalid();
|
||||
const configured = installation.data.metadataGeneration;
|
||||
if (!configured || configured.models.length === 0) {
|
||||
if (configured?.default !== undefined) throw invalid("metadata-generation default does not identify a configured model");
|
||||
return new RestartLoadedMetadataGenerationModels();
|
||||
}
|
||||
if (!configured.default) throw invalid("metadata-generation default is required when models are configured");
|
||||
|
||||
const seen = new Set<string>();
|
||||
for (const model of configured.models) {
|
||||
if (seen.has(model.id)) throw invalid(`metadata-generation model id "${model.id}" is duplicated`);
|
||||
seen.add(model.id);
|
||||
}
|
||||
if (!seen.has(configured.default)) {
|
||||
throw invalid(`metadata-generation default "${configured.default}" is not configured`);
|
||||
}
|
||||
const requiresSecrets = configured.models.some((model) => model.apiKeyEnv !== undefined);
|
||||
let secrets: ReadonlyMap<string, string> = new Map();
|
||||
if (requiresSecrets) {
|
||||
if (!options.secretsFile) throw invalid("metadata-generation keyed models require THT_SECRETS_FILE");
|
||||
try {
|
||||
secrets = loadSecretBundle(options.secretsFile);
|
||||
} catch {
|
||||
throw invalid("metadata-generation secrets are unavailable");
|
||||
}
|
||||
}
|
||||
const models = new Map<string, ResolvedMetadataGenerationModel>();
|
||||
for (const configuredModel of configured.models) {
|
||||
let apiKey: string | undefined;
|
||||
if (configuredModel.apiKeyEnv !== undefined) {
|
||||
apiKey = secrets.get(configuredModel.apiKeyEnv);
|
||||
if (!apiKey) {
|
||||
throw invalid(`metadata-generation model "${configuredModel.id}" secret "${configuredModel.apiKeyEnv}" is missing`);
|
||||
}
|
||||
if (apiKey.length > 16 * 1024 || /\s/u.test(apiKey)) {
|
||||
throw invalid(`metadata-generation model "${configuredModel.id}" secret "${configuredModel.apiKeyEnv}" is unusable`);
|
||||
}
|
||||
}
|
||||
models.set(configuredModel.id, Object.freeze({
|
||||
id: configuredModel.id,
|
||||
provider: configuredModel.litellm.provider,
|
||||
model: configuredModel.litellm.model,
|
||||
...(configuredModel.litellm.disableThinking === true ? { disableThinking: true as const } : {}),
|
||||
...(configuredModel.litellm.endpoint === undefined
|
||||
? {}
|
||||
: { endpoint: Object.freeze({ ...configuredModel.litellm.endpoint }) }),
|
||||
...(configuredModel.apiKeyEnv === undefined
|
||||
? {}
|
||||
: { apiKeyEnv: configuredModel.apiKeyEnv, apiKey }),
|
||||
}));
|
||||
}
|
||||
return new RestartLoadedMetadataGenerationModels(
|
||||
models,
|
||||
configured.default,
|
||||
configured.models.map(({ id, label }) => ({ id, label })),
|
||||
);
|
||||
}
|
||||
@@ -7,6 +7,7 @@ import * as initialMigration from "./migrations/001_workspace_databases.js";
|
||||
import * as catalogTablesMigration from "./migrations/002_catalog_tables.js";
|
||||
import * as catalogSchemaSyncMigration from "./migrations/003_catalog_schema_sync.js";
|
||||
import * as catalogRuntimeSequencePrivilegesMigration from "./migrations/004_catalog_runtime_sequence_privileges.js";
|
||||
import * as descriptionGenerationRunsMigration from "./migrations/005_description_generation_runs.js";
|
||||
|
||||
const connectionString = process.env.THT_CATALOG_MIGRATOR_DATABASE_URL;
|
||||
const host = process.env.THT_CATALOG_DB_HOST;
|
||||
@@ -36,6 +37,7 @@ const provider: MigrationProvider = {
|
||||
"002_catalog_tables": catalogTablesMigration,
|
||||
"003_catalog_schema_sync": catalogSchemaSyncMigration,
|
||||
"004_catalog_runtime_sequence_privileges": catalogRuntimeSequencePrivilegesMigration,
|
||||
"005_description_generation_runs": descriptionGenerationRunsMigration,
|
||||
};
|
||||
},
|
||||
};
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
import { sql, type Kysely } from "kysely";
|
||||
import type { CatalogDatabase } from "../repository.js";
|
||||
|
||||
export async function up(db: Kysely<CatalogDatabase>): Promise<void> {
|
||||
await db.schema.createTable("description_generation_runs")
|
||||
.addColumn("id", "uuid", (column) => column.primaryKey())
|
||||
.addColumn("database_id", "uuid", (column) => column.notNull()
|
||||
.references("workspace_databases.id").onDelete("cascade"))
|
||||
.addColumn("scope", "text", (column) => column.notNull())
|
||||
.addColumn("model_id", "text", (column) => column.notNull())
|
||||
.addColumn("language", "text", (column) => column.notNull())
|
||||
.addColumn("status", "text", (column) => column.notNull())
|
||||
.addColumn("total", "integer", (column) => column.notNull())
|
||||
.addColumn("processed", "integer", (column) => column.notNull().defaultTo(0))
|
||||
.addColumn("generated", "integer", (column) => column.notNull().defaultTo(0))
|
||||
.addColumn("non_generatable", "integer", (column) => column.notNull().defaultTo(0))
|
||||
.addColumn("failed", "integer", (column) => column.notNull().defaultTo(0))
|
||||
.addColumn("created_at", "timestamptz", (column) => column.notNull().defaultTo(sql`now()`))
|
||||
.addColumn("started_at", "timestamptz")
|
||||
.addColumn("updated_at", "timestamptz", (column) => column.notNull().defaultTo(sql`now()`))
|
||||
.addColumn("finished_at", "timestamptz")
|
||||
.addColumn("error_summary", "text")
|
||||
.addCheckConstraint(
|
||||
"description_generation_runs_scope_check",
|
||||
sql`scope in ('selected_columns', 'selected_tables', 'all', 'missing')`,
|
||||
)
|
||||
.addCheckConstraint(
|
||||
"description_generation_runs_model_id_check",
|
||||
sql`model_id ~ '^[a-z][a-z0-9._-]{0,63}$'`,
|
||||
)
|
||||
.addCheckConstraint(
|
||||
"description_generation_runs_language_check",
|
||||
sql`language in ('en', 'it')`,
|
||||
)
|
||||
.addCheckConstraint(
|
||||
"description_generation_runs_status_check",
|
||||
sql`status in (
|
||||
'queued', 'running', 'completed', 'completed_with_errors',
|
||||
'cancelled', 'failed', 'interrupted'
|
||||
)`,
|
||||
)
|
||||
.addCheckConstraint(
|
||||
"description_generation_runs_counters_check",
|
||||
sql`total > 0
|
||||
and processed between 0 and total
|
||||
and generated >= 0
|
||||
and non_generatable >= 0
|
||||
and failed >= 0
|
||||
and generated + non_generatable + failed <= processed`,
|
||||
)
|
||||
.addCheckConstraint(
|
||||
"description_generation_runs_error_summary_check",
|
||||
sql`error_summary is null or char_length(error_summary) between 1 and 2000`,
|
||||
)
|
||||
.execute();
|
||||
await db.schema.createIndex("description_generation_runs_database_created_idx")
|
||||
.on("description_generation_runs")
|
||||
.columns(["database_id", "created_at"])
|
||||
.execute();
|
||||
await sql`CREATE UNIQUE INDEX description_generation_runs_one_active
|
||||
ON description_generation_runs ((true))
|
||||
WHERE status IN ('queued', 'running')`.execute(db);
|
||||
|
||||
await db.schema.createTable("description_generation_events")
|
||||
.addColumn("run_id", "uuid", (column) => column.notNull()
|
||||
.references("description_generation_runs.id").onDelete("cascade"))
|
||||
.addColumn("sequence", "integer", (column) => column.notNull())
|
||||
.addColumn("level", "text", (column) => column.notNull())
|
||||
.addColumn("message", "text", (column) => column.notNull())
|
||||
.addColumn("created_at", "timestamptz", (column) => column.notNull().defaultTo(sql`now()`))
|
||||
.addPrimaryKeyConstraint("description_generation_events_pkey", ["run_id", "sequence"])
|
||||
.addCheckConstraint("description_generation_events_sequence_check", sql`sequence > 0`)
|
||||
.addCheckConstraint(
|
||||
"description_generation_events_level_check",
|
||||
sql`level in ('info', 'warning', 'error')`,
|
||||
)
|
||||
.addCheckConstraint(
|
||||
"description_generation_events_message_check",
|
||||
sql`char_length(message) between 1 and 2000`,
|
||||
)
|
||||
.execute();
|
||||
}
|
||||
|
||||
export async function down(db: Kysely<CatalogDatabase>): Promise<void> {
|
||||
await db.schema.dropTable("description_generation_events").execute();
|
||||
await db.schema.dropTable("description_generation_runs").execute();
|
||||
}
|
||||
@@ -0,0 +1,150 @@
|
||||
import { spawn } from "node:child_process";
|
||||
import { z } from "zod";
|
||||
import type { ResolvedMetadataGenerationModel } from "./metadata-generation-models.js";
|
||||
|
||||
const MAX_HELPER_OUTPUT_BYTES = 64 * 1024;
|
||||
const helperOutputSchema = z.discriminatedUnion("ok", [
|
||||
z.object({ ok: z.literal(true), content: z.string() }).strict(),
|
||||
z.object({ ok: z.literal(false), error: z.literal("provider_failure") }).strict(),
|
||||
]);
|
||||
|
||||
export interface ModelCompletionMessage {
|
||||
role: "system" | "user";
|
||||
content: string;
|
||||
}
|
||||
|
||||
export interface ModelCompletionRequest {
|
||||
model: ResolvedMetadataGenerationModel;
|
||||
messages: readonly ModelCompletionMessage[];
|
||||
signal: AbortSignal;
|
||||
}
|
||||
|
||||
/** The provider boundary used by Description Generation. */
|
||||
export interface ModelCompleter {
|
||||
complete(request: ModelCompletionRequest): Promise<string>;
|
||||
}
|
||||
|
||||
export class ModelCompletionProviderError extends Error {
|
||||
constructor() {
|
||||
super("model completion failed");
|
||||
this.name = "ModelCompletionProviderError";
|
||||
}
|
||||
}
|
||||
|
||||
export class ModelCompletionCancelledError extends Error {
|
||||
constructor() {
|
||||
super("model completion cancelled");
|
||||
this.name = "ModelCompletionCancelledError";
|
||||
}
|
||||
}
|
||||
|
||||
export class PythonModelCompleter implements ModelCompleter {
|
||||
constructor(private readonly options: {
|
||||
pythonExecutable: string;
|
||||
cwd: string;
|
||||
helperModule?: string;
|
||||
timeoutMs?: number;
|
||||
terminationGraceMs?: number;
|
||||
}) {}
|
||||
|
||||
async complete(request: ModelCompletionRequest): Promise<string> {
|
||||
if (request.signal.aborted) throw new ModelCompletionCancelledError();
|
||||
const payload = {
|
||||
model: `${request.model.provider}/${request.model.model}`,
|
||||
...(request.model.apiKey === undefined ? {} : { api_key: request.model.apiKey }),
|
||||
messages: request.messages.map((message) => ({ ...message })),
|
||||
...(request.model.endpoint?.baseUrl === undefined
|
||||
? {}
|
||||
: { api_base: request.model.endpoint.baseUrl }),
|
||||
...(request.model.endpoint?.apiVersion === undefined
|
||||
? {}
|
||||
: { api_version: request.model.endpoint.apiVersion }),
|
||||
...(request.model.disableThinking === true ? { disable_thinking: true } : {}),
|
||||
};
|
||||
|
||||
return await new Promise<string>((resolve, reject) => {
|
||||
const child = spawn(
|
||||
this.options.pythonExecutable,
|
||||
["-m", this.options.helperModule ?? "tht.internal.litellm_completion"],
|
||||
{
|
||||
cwd: this.options.cwd,
|
||||
stdio: ["pipe", "pipe", "pipe"],
|
||||
},
|
||||
);
|
||||
let stdout = "";
|
||||
let settled = false;
|
||||
let timeout: ReturnType<typeof setTimeout> | undefined;
|
||||
let killFallback: ReturnType<typeof setTimeout> | undefined;
|
||||
let terminatingWith: Error | undefined;
|
||||
const cleanup = () => {
|
||||
if (timeout) clearTimeout(timeout);
|
||||
if (killFallback) clearTimeout(killFallback);
|
||||
request.signal.removeEventListener("abort", cancel);
|
||||
};
|
||||
const fail = (error: Error = new ModelCompletionProviderError()) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
cleanup();
|
||||
reject(error);
|
||||
};
|
||||
const terminate = (error: Error) => {
|
||||
if (settled || terminatingWith) return;
|
||||
terminatingWith = error;
|
||||
if (timeout) clearTimeout(timeout);
|
||||
try {
|
||||
child.kill("SIGTERM");
|
||||
} catch {
|
||||
fail(error);
|
||||
return;
|
||||
}
|
||||
killFallback = setTimeout(() => {
|
||||
if (settled || child.exitCode !== null || child.signalCode !== null) return;
|
||||
try {
|
||||
child.kill("SIGKILL");
|
||||
} catch {
|
||||
fail(error);
|
||||
}
|
||||
}, this.options.terminationGraceMs ?? 250);
|
||||
};
|
||||
const cancel = () => {
|
||||
terminate(new ModelCompletionCancelledError());
|
||||
};
|
||||
timeout = setTimeout(() => {
|
||||
terminate(new ModelCompletionProviderError());
|
||||
}, this.options.timeoutMs ?? 120_000);
|
||||
request.signal.addEventListener("abort", cancel, { once: true });
|
||||
if (request.signal.aborted) cancel();
|
||||
|
||||
child.stdout.setEncoding("utf8");
|
||||
child.stdout.on("data", (chunk: string) => {
|
||||
if (terminatingWith) return;
|
||||
stdout += chunk;
|
||||
if (Buffer.byteLength(stdout, "utf8") > MAX_HELPER_OUTPUT_BYTES) {
|
||||
terminate(new ModelCompletionProviderError());
|
||||
}
|
||||
});
|
||||
// Helper and provider diagnostics are deliberately not copied into application logs.
|
||||
child.stderr.resume();
|
||||
child.once("error", () => fail(terminatingWith ?? new ModelCompletionProviderError()));
|
||||
child.once("close", (code) => {
|
||||
if (settled) return;
|
||||
if (terminatingWith) return fail(terminatingWith);
|
||||
try {
|
||||
if (code !== 0) return fail();
|
||||
const output = helperOutputSchema.parse(JSON.parse(stdout));
|
||||
if (!output.ok) return fail();
|
||||
settled = true;
|
||||
cleanup();
|
||||
resolve(output.content);
|
||||
} catch {
|
||||
fail();
|
||||
}
|
||||
});
|
||||
child.stdin.once("error", () => {
|
||||
if (terminatingWith) return;
|
||||
terminate(new ModelCompletionProviderError());
|
||||
});
|
||||
child.stdin.end(JSON.stringify(payload));
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -1,22 +1,34 @@
|
||||
import { CatalogOperationInProgressError } from "./types.js";
|
||||
|
||||
/** Serializes connection tests, synchronization, and metadata cleanup for each catalog database. */
|
||||
/** Serializes mutable catalog operations for each Workspace Database. */
|
||||
export class CatalogOperationCoordinator {
|
||||
private readonly active = new Set<string>();
|
||||
private readonly active = new Map<
|
||||
string,
|
||||
{ token: symbol; owner: "catalog_operation" | "description_generation" }
|
||||
>();
|
||||
|
||||
reserve(databaseId: string): () => void {
|
||||
reserve(
|
||||
databaseId: string,
|
||||
owner: "catalog_operation" | "description_generation" = "catalog_operation",
|
||||
): () => void {
|
||||
if (this.active.has(databaseId)) {
|
||||
throw new CatalogOperationInProgressError("A database operation is already in progress");
|
||||
}
|
||||
this.active.add(databaseId);
|
||||
const token = Symbol(databaseId);
|
||||
this.active.set(databaseId, { token, owner });
|
||||
let released = false;
|
||||
return () => {
|
||||
if (released) return;
|
||||
released = true;
|
||||
this.active.delete(databaseId);
|
||||
if (this.active.get(databaseId)?.token === token) this.active.delete(databaseId);
|
||||
};
|
||||
}
|
||||
|
||||
/** Administrative recovery for a reservation whose owning local operation is no longer live. */
|
||||
releaseStale(databaseId: string, owner: "description_generation"): void {
|
||||
if (this.active.get(databaseId)?.owner === owner) this.active.delete(databaseId);
|
||||
}
|
||||
|
||||
async run<T>(databaseId: string, operation: () => Promise<T>): Promise<T> {
|
||||
const release = this.reserve(databaseId);
|
||||
try {
|
||||
|
||||
@@ -180,13 +180,17 @@ export class ConcreteCatalogPostgresAccess implements CatalogPostgresAccess {
|
||||
materialized.release();
|
||||
};
|
||||
const abort = () => { void close(); };
|
||||
signal.addEventListener("abort", abort, { once: true });
|
||||
|
||||
try {
|
||||
if (signal.aborted) throw new CatalogConnectorError("PostgreSQL connector aborted");
|
||||
const passwordFile = materialized.files.get(CATALOG_SECRET_IDS.password);
|
||||
if (!passwordFile) throw new CatalogConnectorError("Database password is not configured");
|
||||
const password = await readFile(passwordFile, "utf8");
|
||||
if (signal.aborted) throw new CatalogConnectorError("PostgreSQL connector aborted");
|
||||
const tlsCaFile = materialized.files.get(CATALOG_SECRET_IDS.tlsCa);
|
||||
const tlsCa = tlsCaFile ? await readFile(tlsCaFile, "utf8") : undefined;
|
||||
if (signal.aborted) throw new CatalogConnectorError("PostgreSQL connector aborted");
|
||||
let host: string;
|
||||
let port: number;
|
||||
|
||||
@@ -243,8 +247,8 @@ export class ConcreteCatalogPostgresAccess implements CatalogPostgresAccess {
|
||||
connectionTimeoutMillis: this.connectTimeoutMs,
|
||||
...(stream ? { stream: () => stream } : {}),
|
||||
});
|
||||
signal.addEventListener("abort", abort, { once: true });
|
||||
await client.connect();
|
||||
if (signal.aborted) throw new CatalogConnectorError("PostgreSQL connector aborted");
|
||||
return {
|
||||
query: async (sql, values) => await client!.query(sql, [...values]),
|
||||
end: close,
|
||||
|
||||
@@ -15,7 +15,10 @@ import {
|
||||
CatalogConflictError,
|
||||
CatalogConnectorError,
|
||||
CatalogUnavailableError,
|
||||
DescriptionGenerationRunActiveError,
|
||||
type CatalogColumn,
|
||||
type CatalogDescriptionConsolidationCounts,
|
||||
type CatalogDescriptionTarget,
|
||||
type CatalogDatabaseMetadataDeleteTarget,
|
||||
type CatalogMetadataDeleteCounts,
|
||||
type CatalogRelationship,
|
||||
@@ -31,6 +34,10 @@ import {
|
||||
type DatabaseBinding,
|
||||
type DatabaseConfigurationInput,
|
||||
type DatabaseTestResult,
|
||||
type DescriptionGenerationEvent,
|
||||
type DescriptionGenerationRun,
|
||||
type DescriptionGenerationRunUpdate,
|
||||
type DescriptionGenerationScope,
|
||||
type ObservedCatalogTable,
|
||||
type ObservedSchemaSnapshot,
|
||||
type TableSyncRepositoryResult,
|
||||
@@ -131,6 +138,33 @@ interface CatalogRelationshipColumnTable {
|
||||
targetColumnId: string;
|
||||
}
|
||||
|
||||
interface DescriptionGenerationRunTable {
|
||||
id: string;
|
||||
databaseId: string;
|
||||
scope: DescriptionGenerationScope;
|
||||
modelId: string;
|
||||
language: DescriptionGenerationRun["language"];
|
||||
status: DescriptionGenerationRun["status"];
|
||||
total: number;
|
||||
processed: number;
|
||||
generated: number;
|
||||
nonGeneratable: number;
|
||||
failed: number;
|
||||
createdAt: Timestamp;
|
||||
startedAt: Timestamp | null;
|
||||
updatedAt: Timestamp;
|
||||
finishedAt: Timestamp | null;
|
||||
errorSummary: string | null;
|
||||
}
|
||||
|
||||
interface DescriptionGenerationEventTable {
|
||||
runId: string;
|
||||
sequence: number;
|
||||
level: DescriptionGenerationEvent["level"];
|
||||
message: string;
|
||||
createdAt: Timestamp;
|
||||
}
|
||||
|
||||
interface CatalogSyncRunTable {
|
||||
id: string;
|
||||
databaseId: string;
|
||||
@@ -176,6 +210,8 @@ export interface CatalogDatabase {
|
||||
catalogColumns: CatalogColumnTable;
|
||||
catalogRelationships: CatalogRelationshipTable;
|
||||
catalogRelationshipColumns: CatalogRelationshipColumnTable;
|
||||
descriptionGenerationRuns: DescriptionGenerationRunTable;
|
||||
descriptionGenerationEvents: DescriptionGenerationEventTable;
|
||||
catalogSyncRuns: CatalogSyncRunTable;
|
||||
catalogSyncEvents: CatalogSyncEventTable;
|
||||
}
|
||||
@@ -281,6 +317,25 @@ function serializeSyncEvent(row: Selectable<CatalogSyncEventTable>): CatalogSync
|
||||
return { ...row, id: Number(row.id), data: row.data ?? {}, createdAt: new Date(row.createdAt).toISOString() };
|
||||
}
|
||||
|
||||
function serializeDescriptionGenerationRun(
|
||||
row: Selectable<DescriptionGenerationRunTable>,
|
||||
): DescriptionGenerationRun {
|
||||
const stamp = (value: Date | string | null) => value === null ? null : new Date(value).toISOString();
|
||||
return {
|
||||
...row,
|
||||
createdAt: new Date(row.createdAt).toISOString(),
|
||||
startedAt: stamp(row.startedAt),
|
||||
updatedAt: new Date(row.updatedAt).toISOString(),
|
||||
finishedAt: stamp(row.finishedAt),
|
||||
};
|
||||
}
|
||||
|
||||
function serializeDescriptionGenerationEvent(
|
||||
row: Selectable<DescriptionGenerationEventTable>,
|
||||
): DescriptionGenerationEvent {
|
||||
return { ...row, createdAt: new Date(row.createdAt).toISOString() };
|
||||
}
|
||||
|
||||
function bindingValues(databaseId: string, binding: DatabaseBinding) {
|
||||
return {
|
||||
databaseId,
|
||||
@@ -520,6 +575,202 @@ export class KyselyCatalogRepository implements CatalogRepository {
|
||||
return row ? await this.getColumn(databaseId, tableId, row.id) : undefined;
|
||||
}
|
||||
|
||||
async consolidateGeneratedDescriptions(
|
||||
databaseId: string,
|
||||
target: CatalogDescriptionTarget,
|
||||
targetIds: readonly string[],
|
||||
): Promise<CatalogDescriptionConsolidationCounts | undefined> {
|
||||
const selectedTargetIds = [...new Set(targetIds)];
|
||||
if (selectedTargetIds.length === 0) return undefined;
|
||||
return await this.db.transaction().execute(async (trx) => {
|
||||
const database = await trx.selectFrom("workspaceDatabases").select("id")
|
||||
.where("id", "=", databaseId).forUpdate().executeTakeFirst();
|
||||
if (!database) return undefined;
|
||||
if (target === "tables") {
|
||||
const rows = await trx.selectFrom("catalogTables")
|
||||
.select(["id", "generatedDescription"])
|
||||
.where("databaseId", "=", databaseId)
|
||||
.where("id", "in", selectedTargetIds)
|
||||
.orderBy("id")
|
||||
.forUpdate()
|
||||
.execute();
|
||||
if (rows.length !== selectedTargetIds.length) return undefined;
|
||||
const copiedIds = rows
|
||||
.filter((row) => Boolean(row.generatedDescription?.trim()))
|
||||
.map((row) => row.id);
|
||||
if (copiedIds.length > 0) {
|
||||
await trx.updateTable("catalogTables").set({
|
||||
description: sql`generated_description`,
|
||||
version: sql`version + 1`,
|
||||
updatedAt: sql`now()`,
|
||||
}).where("id", "in", copiedIds).execute();
|
||||
}
|
||||
return {
|
||||
copied: copiedIds.length,
|
||||
skipped: selectedTargetIds.length - copiedIds.length,
|
||||
};
|
||||
}
|
||||
|
||||
const tableRows = await trx.selectFrom("catalogTables").select("id")
|
||||
.where("databaseId", "=", databaseId).execute();
|
||||
const rows = tableRows.length === 0 ? [] : await trx.selectFrom("catalogColumns")
|
||||
.select(["id", "generatedDescription"])
|
||||
.where("tableId", "in", tableRows.map((table) => table.id))
|
||||
.where("id", "in", selectedTargetIds)
|
||||
.orderBy("id")
|
||||
.forUpdate()
|
||||
.execute();
|
||||
if (rows.length !== selectedTargetIds.length) return undefined;
|
||||
const copiedIds = rows
|
||||
.filter((row) => Boolean(row.generatedDescription?.trim()))
|
||||
.map((row) => row.id);
|
||||
if (copiedIds.length > 0) {
|
||||
await trx.updateTable("catalogColumns").set({
|
||||
description: sql`generated_description`,
|
||||
version: sql`version + 1`,
|
||||
updatedAt: sql`now()`,
|
||||
}).where("id", "in", copiedIds).execute();
|
||||
}
|
||||
return {
|
||||
copied: copiedIds.length,
|
||||
skipped: selectedTargetIds.length - copiedIds.length,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
async createDescriptionGenerationRun(
|
||||
databaseId: string,
|
||||
scope: DescriptionGenerationScope,
|
||||
modelId: string,
|
||||
language: DescriptionGenerationRun["language"],
|
||||
total: number,
|
||||
): Promise<DescriptionGenerationRun> {
|
||||
try {
|
||||
const row = await this.db.insertInto("descriptionGenerationRuns").values({
|
||||
id: randomUUID(),
|
||||
databaseId,
|
||||
scope,
|
||||
modelId,
|
||||
language,
|
||||
status: "queued",
|
||||
total,
|
||||
processed: 0,
|
||||
generated: 0,
|
||||
nonGeneratable: 0,
|
||||
failed: 0,
|
||||
startedAt: null,
|
||||
finishedAt: null,
|
||||
errorSummary: null,
|
||||
}).returningAll().executeTakeFirstOrThrow();
|
||||
return serializeDescriptionGenerationRun(row);
|
||||
} catch (error: any) {
|
||||
if (error?.code === "23505"
|
||||
&& error?.constraint === "description_generation_runs_one_active") {
|
||||
throw new DescriptionGenerationRunActiveError(
|
||||
"A description generation run is already active",
|
||||
);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async getDescriptionGenerationRun(
|
||||
runId: string,
|
||||
): Promise<DescriptionGenerationRun | undefined> {
|
||||
const row = await this.db.selectFrom("descriptionGenerationRuns")
|
||||
.selectAll()
|
||||
.where("id", "=", runId)
|
||||
.executeTakeFirst();
|
||||
return row ? serializeDescriptionGenerationRun(row) : undefined;
|
||||
}
|
||||
|
||||
async listDescriptionGenerationRuns(limit = 50): Promise<DescriptionGenerationRun[]> {
|
||||
const rows = await this.db.selectFrom("descriptionGenerationRuns")
|
||||
.selectAll()
|
||||
.orderBy("createdAt", "desc")
|
||||
.orderBy("id", "desc")
|
||||
.limit(limit)
|
||||
.execute();
|
||||
return rows.map(serializeDescriptionGenerationRun);
|
||||
}
|
||||
|
||||
async getActiveDescriptionGenerationRun(): Promise<DescriptionGenerationRun | undefined> {
|
||||
const row = await this.db.selectFrom("descriptionGenerationRuns")
|
||||
.selectAll()
|
||||
.where("status", "in", ["queued", "running"])
|
||||
.orderBy("createdAt", "desc")
|
||||
.executeTakeFirst();
|
||||
return row ? serializeDescriptionGenerationRun(row) : undefined;
|
||||
}
|
||||
|
||||
async interruptActiveDescriptionGenerationRuns(
|
||||
errorSummary: string,
|
||||
): Promise<DescriptionGenerationRun[]> {
|
||||
const rows = await this.db.updateTable("descriptionGenerationRuns")
|
||||
.set({
|
||||
status: "interrupted",
|
||||
finishedAt: sql`now()`,
|
||||
updatedAt: sql`now()`,
|
||||
errorSummary,
|
||||
})
|
||||
.where("status", "in", ["queued", "running"])
|
||||
.returningAll()
|
||||
.execute();
|
||||
return rows.map(serializeDescriptionGenerationRun);
|
||||
}
|
||||
|
||||
async updateDescriptionGenerationRun(
|
||||
runId: string,
|
||||
update: DescriptionGenerationRunUpdate,
|
||||
): Promise<DescriptionGenerationRun | undefined> {
|
||||
const values: any = { ...update, updatedAt: sql`now()` };
|
||||
const row = await this.db.updateTable("descriptionGenerationRuns")
|
||||
.set(values)
|
||||
.where("id", "=", runId)
|
||||
.returningAll()
|
||||
.executeTakeFirst();
|
||||
return row ? serializeDescriptionGenerationRun(row) : undefined;
|
||||
}
|
||||
|
||||
async appendDescriptionGenerationEvent(
|
||||
runId: string,
|
||||
level: DescriptionGenerationEvent["level"],
|
||||
message: string,
|
||||
): Promise<DescriptionGenerationEvent> {
|
||||
return await this.db.transaction().execute(async (trx) => {
|
||||
const run = await trx.selectFrom("descriptionGenerationRuns")
|
||||
.select("id")
|
||||
.where("id", "=", runId)
|
||||
.forUpdate()
|
||||
.executeTakeFirst();
|
||||
if (!run) throw new CatalogConflictError("Description Generation Run does not exist");
|
||||
const current = await trx.selectFrom("descriptionGenerationEvents")
|
||||
.select(sql<number>`coalesce(max(sequence), 0)::int`.as("sequence"))
|
||||
.where("runId", "=", runId)
|
||||
.executeTakeFirst();
|
||||
const row = await trx.insertInto("descriptionGenerationEvents").values({
|
||||
runId,
|
||||
sequence: Number(current?.sequence ?? 0) + 1,
|
||||
level,
|
||||
message,
|
||||
}).returningAll().executeTakeFirstOrThrow();
|
||||
return serializeDescriptionGenerationEvent(row);
|
||||
});
|
||||
}
|
||||
|
||||
async listDescriptionGenerationEvents(
|
||||
runId: string,
|
||||
afterSequence = 0,
|
||||
): Promise<DescriptionGenerationEvent[]> {
|
||||
const rows = await this.db.selectFrom("descriptionGenerationEvents")
|
||||
.selectAll()
|
||||
.where("runId", "=", runId)
|
||||
.where("sequence", ">", afterSequence)
|
||||
.orderBy("sequence")
|
||||
.execute();
|
||||
return rows.map(serializeDescriptionGenerationEvent);
|
||||
}
|
||||
|
||||
async listRelationships(databaseId: string): Promise<CatalogRelationship[]> {
|
||||
const rows = await this.db.selectFrom("catalogRelationships as relationship")
|
||||
.innerJoin("catalogTables as sourceTable", "sourceTable.id", "relationship.sourceTableId")
|
||||
@@ -1102,6 +1353,15 @@ export class UnavailableCatalogRepository implements CatalogRepository {
|
||||
async listColumns(): Promise<CatalogColumn[]> { return this.fail(); }
|
||||
async getColumn(): Promise<CatalogColumn | undefined> { return this.fail(); }
|
||||
async updateColumnMetadata(): Promise<CatalogColumn | undefined> { return this.fail(); }
|
||||
async consolidateGeneratedDescriptions(): Promise<CatalogDescriptionConsolidationCounts | undefined> { return this.fail(); }
|
||||
async createDescriptionGenerationRun(): Promise<DescriptionGenerationRun> { return this.fail(); }
|
||||
async getDescriptionGenerationRun(): Promise<DescriptionGenerationRun | undefined> { return this.fail(); }
|
||||
async listDescriptionGenerationRuns(): Promise<DescriptionGenerationRun[]> { return this.fail(); }
|
||||
async getActiveDescriptionGenerationRun(): Promise<DescriptionGenerationRun | undefined> { return this.fail(); }
|
||||
async interruptActiveDescriptionGenerationRuns(): Promise<DescriptionGenerationRun[]> { return this.fail(); }
|
||||
async updateDescriptionGenerationRun(): Promise<DescriptionGenerationRun | undefined> { return this.fail(); }
|
||||
async appendDescriptionGenerationEvent(): Promise<DescriptionGenerationEvent> { return this.fail(); }
|
||||
async listDescriptionGenerationEvents(): Promise<DescriptionGenerationEvent[]> { return this.fail(); }
|
||||
async listRelationships(): Promise<CatalogRelationship[]> { return this.fail(); }
|
||||
async deleteDatabaseMetadata(): Promise<CatalogMetadataDeleteCounts | undefined> { return this.fail(); }
|
||||
async deleteTableMetadata(): Promise<CatalogMetadataDeleteCounts | undefined> { return this.fail(); }
|
||||
|
||||
@@ -135,6 +135,7 @@ export interface CatalogRelationship {
|
||||
|
||||
export type CatalogDatabaseMetadataDeleteTarget = "tables" | "relationships";
|
||||
export type CatalogTableMetadataDeleteTarget = "columns" | "relationships";
|
||||
export type CatalogDescriptionTarget = "tables" | "columns";
|
||||
|
||||
export interface CatalogMetadataDeleteCounts {
|
||||
tables: number;
|
||||
@@ -142,6 +143,63 @@ export interface CatalogMetadataDeleteCounts {
|
||||
relationships: number;
|
||||
}
|
||||
|
||||
export interface CatalogDescriptionConsolidationCounts {
|
||||
copied: number;
|
||||
skipped: number;
|
||||
}
|
||||
|
||||
export type DescriptionGenerationScope =
|
||||
| "selected_columns"
|
||||
| "selected_tables"
|
||||
| "all"
|
||||
| "missing";
|
||||
export type DescriptionGenerationStatus =
|
||||
| "queued"
|
||||
| "running"
|
||||
| "completed"
|
||||
| "completed_with_errors"
|
||||
| "cancelled"
|
||||
| "failed"
|
||||
| "interrupted";
|
||||
|
||||
export interface DescriptionGenerationRun {
|
||||
id: string;
|
||||
databaseId: string;
|
||||
scope: DescriptionGenerationScope;
|
||||
modelId: string;
|
||||
language: "en" | "it";
|
||||
status: DescriptionGenerationStatus;
|
||||
total: number;
|
||||
processed: number;
|
||||
generated: number;
|
||||
nonGeneratable: number;
|
||||
failed: number;
|
||||
createdAt: string;
|
||||
startedAt: string | null;
|
||||
updatedAt: string;
|
||||
finishedAt: string | null;
|
||||
errorSummary: string | null;
|
||||
}
|
||||
|
||||
export interface DescriptionGenerationRunUpdate {
|
||||
status?: DescriptionGenerationStatus;
|
||||
processed?: number;
|
||||
generated?: number;
|
||||
nonGeneratable?: number;
|
||||
failed?: number;
|
||||
startedAt?: string | null;
|
||||
finishedAt?: string | null;
|
||||
errorSummary?: string | null;
|
||||
}
|
||||
|
||||
export interface DescriptionGenerationEvent {
|
||||
runId: string;
|
||||
sequence: number;
|
||||
level: "info" | "warning" | "error";
|
||||
message: string;
|
||||
createdAt: string;
|
||||
}
|
||||
|
||||
export interface ObservedRelationshipColumn {
|
||||
position: number;
|
||||
sourceColumnName: string;
|
||||
@@ -292,6 +350,37 @@ export interface CatalogRepository {
|
||||
description: string | null,
|
||||
generatedDescription: string | null,
|
||||
): Promise<CatalogColumn | undefined>;
|
||||
consolidateGeneratedDescriptions(
|
||||
databaseId: string,
|
||||
target: CatalogDescriptionTarget,
|
||||
targetIds: readonly string[],
|
||||
): Promise<CatalogDescriptionConsolidationCounts | undefined>;
|
||||
createDescriptionGenerationRun(
|
||||
databaseId: string,
|
||||
scope: DescriptionGenerationScope,
|
||||
modelId: string,
|
||||
language: DescriptionGenerationRun["language"],
|
||||
total: number,
|
||||
): Promise<DescriptionGenerationRun>;
|
||||
getDescriptionGenerationRun(runId: string): Promise<DescriptionGenerationRun | undefined>;
|
||||
listDescriptionGenerationRuns(limit?: number): Promise<DescriptionGenerationRun[]>;
|
||||
getActiveDescriptionGenerationRun(): Promise<DescriptionGenerationRun | undefined>;
|
||||
interruptActiveDescriptionGenerationRuns(
|
||||
errorSummary: string,
|
||||
): Promise<DescriptionGenerationRun[]>;
|
||||
updateDescriptionGenerationRun(
|
||||
runId: string,
|
||||
update: DescriptionGenerationRunUpdate,
|
||||
): Promise<DescriptionGenerationRun | undefined>;
|
||||
appendDescriptionGenerationEvent(
|
||||
runId: string,
|
||||
level: DescriptionGenerationEvent["level"],
|
||||
message: string,
|
||||
): Promise<DescriptionGenerationEvent>;
|
||||
listDescriptionGenerationEvents(
|
||||
runId: string,
|
||||
afterSequence?: number,
|
||||
): Promise<DescriptionGenerationEvent[]>;
|
||||
listRelationships(databaseId: string): Promise<CatalogRelationship[]>;
|
||||
deleteDatabaseMetadata(
|
||||
databaseIds: readonly string[],
|
||||
@@ -347,6 +436,7 @@ export interface CatalogRepository {
|
||||
}
|
||||
|
||||
export class CatalogConflictError extends Error {}
|
||||
export class DescriptionGenerationRunActiveError extends CatalogConflictError {}
|
||||
export class CatalogUnavailableError extends Error {}
|
||||
export class CatalogOperationInProgressError extends Error {}
|
||||
export class CatalogConnectorError extends Error {}
|
||||
|
||||
@@ -31,6 +31,7 @@ export interface AppConfig {
|
||||
ollamaEnsureTimeoutMs: number;
|
||||
piManagementTimeoutMs: number;
|
||||
secretsFile?: string;
|
||||
installationConfigFile?: string;
|
||||
piAuthFile?: string;
|
||||
secretFiles: Readonly<Record<string, string | undefined>>;
|
||||
modelApiKeyFile?: string;
|
||||
@@ -322,6 +323,13 @@ export function loadConfig(
|
||||
secretsFile.trim() !== secretsFile || secretsFile.length === 0 || secretsFile.includes("\0")
|
||||
|| !path.isAbsolute(secretsFile)
|
||||
)) throw new Error("secret bundle configuration is invalid");
|
||||
const installationConfigFile = env.THT_INSTALLATION_CONFIG_FILE;
|
||||
if (installationConfigFile !== undefined && (
|
||||
installationConfigFile.trim() !== installationConfigFile
|
||||
|| installationConfigFile.length === 0
|
||||
|| installationConfigFile.includes("\0")
|
||||
|| !path.isAbsolute(installationConfigFile)
|
||||
)) throw new Error("installation configuration is invalid");
|
||||
const piAuthFile = env.THT_PI_AUTH_FILE;
|
||||
if (piAuthFile !== undefined && (
|
||||
piAuthFile.trim() !== piAuthFile || piAuthFile.length === 0 || piAuthFile.includes("\0")
|
||||
@@ -404,6 +412,7 @@ export function loadConfig(
|
||||
ollamaEnsureTimeoutMs: Number(env.OLLAMA_ENSURE_TIMEOUT_MS ?? 60000),
|
||||
piManagementTimeoutMs: piManagementTimeout(env.PI_MANAGEMENT_TIMEOUT_MS),
|
||||
secretsFile,
|
||||
installationConfigFile,
|
||||
piAuthFile,
|
||||
secretFiles,
|
||||
modelApiKeyFile,
|
||||
|
||||
@@ -8,12 +8,19 @@ import {
|
||||
isUsableAuthenticationSecret,
|
||||
} from "../auth/secret-policy.js";
|
||||
|
||||
/** Credential names that metadata-generation model entries may reference. */
|
||||
export const METADATA_GENERATION_SECRET_KEYS = Object.freeze([
|
||||
"THT_METADATA_API_KEY", "ANTHROPIC_API_KEY", "AZURE_API_KEY", "GEMINI_API_KEY",
|
||||
"DEEPSEEK_API_KEY", "OPENAI_API_KEY", "OPENROUTER_API_KEY", "ZAI_API_KEY",
|
||||
] as const);
|
||||
|
||||
/** Keys accepted by the deployment bundle. Keep this list intentionally explicit. */
|
||||
export const SECRET_BUNDLE_KEYS = Object.freeze([
|
||||
"THT_MODEL_API_KEY", "THT_DWH_API_KEY", "THT_VEC_API_KEY", "THT_VEC_WRITE_API_KEY",
|
||||
"THT_CA", "THT_SSL_CA", "THT_VECTOR_BOOTSTRAP_PASSWORD", "THT_VECTOR_MIGRATOR_PASSWORD",
|
||||
"THT_VECTOR_READER_PASSWORD", "THT_VECTOR_WRITER_PASSWORD", "PI_PROVIDER_API_KEY",
|
||||
"THT_OIDC_CLIENT_SECRET", "THT_AUTHENTIK_API_TOKEN",
|
||||
...METADATA_GENERATION_SECRET_KEYS,
|
||||
] as const);
|
||||
|
||||
const ALLOWED = new Set<string>(SECRET_BUNDLE_KEYS);
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
import type { FastifyInstance, FastifyReply, FastifyRequest } from "fastify";
|
||||
import { z } from "zod";
|
||||
import { isPrincipalContext, requirePermission } from "../auth/authorization.js";
|
||||
import type { CatalogOperationCoordinator } from "../catalog/operation-coordinator.js";
|
||||
import {
|
||||
CatalogOperationInProgressError,
|
||||
CatalogUnavailableError,
|
||||
type CatalogRepository,
|
||||
} from "../catalog/types.js";
|
||||
|
||||
const idSchema = z.uuid();
|
||||
const consolidationSchema = z.object({
|
||||
target: z.enum(["tables", "columns"]),
|
||||
targetIds: z.array(idSchema).min(1).max(10_000),
|
||||
}).strict();
|
||||
|
||||
function manage(request: FastifyRequest, reply: FastifyReply) {
|
||||
return isPrincipalContext(requirePermission(request, reply, "database.manage"));
|
||||
}
|
||||
|
||||
function safeError(reply: FastifyReply, error: unknown) {
|
||||
if (error instanceof CatalogUnavailableError) {
|
||||
return reply.code(503).send({ code: "catalog_unavailable", message: "Database catalog is unavailable." });
|
||||
}
|
||||
if (error instanceof CatalogOperationInProgressError) {
|
||||
return reply.code(409).send({
|
||||
code: "database_operation_in_progress",
|
||||
message: "A database operation is already in progress.",
|
||||
});
|
||||
}
|
||||
if (error instanceof z.ZodError) {
|
||||
return reply.code(400).send({
|
||||
code: "description_consolidation_invalid",
|
||||
message: "Description consolidation request is invalid.",
|
||||
});
|
||||
}
|
||||
return reply.code(500).send({
|
||||
code: "description_consolidation_failed",
|
||||
message: "Description consolidation failed.",
|
||||
});
|
||||
}
|
||||
|
||||
export function catalogDescriptionConsolidationRoutes(
|
||||
app: FastifyInstance,
|
||||
deps: { repository: CatalogRepository; operations: CatalogOperationCoordinator },
|
||||
): void {
|
||||
app.post("/catalog/databases/:databaseId/descriptions/consolidate", async (request, reply) => {
|
||||
if (!manage(request, reply)) return reply;
|
||||
try {
|
||||
const databaseId = idSchema.parse((request.params as { databaseId?: unknown }).databaseId);
|
||||
const input = consolidationSchema.parse(request.body);
|
||||
const targetIds = [...new Set(input.targetIds)];
|
||||
const result = await deps.operations.run(
|
||||
databaseId,
|
||||
async () => await deps.repository.consolidateGeneratedDescriptions(
|
||||
databaseId,
|
||||
input.target,
|
||||
targetIds,
|
||||
),
|
||||
);
|
||||
if (!result) {
|
||||
return reply.code(404).send({
|
||||
code: "catalog_target_not_found",
|
||||
message: "The database or one or more selected catalog targets were not found.",
|
||||
});
|
||||
}
|
||||
return result;
|
||||
} catch (error) {
|
||||
return safeError(reply, error);
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,347 @@
|
||||
import type { FastifyInstance, FastifyReply, FastifyRequest } from "fastify";
|
||||
import { z } from "zod";
|
||||
import { isPrincipalContext, requirePermission } from "../auth/authorization.js";
|
||||
import {
|
||||
DescriptionGenerationDuplicateTargetIdsError,
|
||||
DescriptionGenerationNoEligibleTargetsError,
|
||||
DescriptionGenerationRunLiveError,
|
||||
DescriptionGenerationTargetIdsRequiredError,
|
||||
DescriptionGenerationTargetNotFoundError,
|
||||
DescriptionGenerationWorkspaceUnavailableError,
|
||||
type DescriptionGenerationWorker,
|
||||
} from "../catalog/description-generation-worker.js";
|
||||
import { MetadataGenerationModelUnavailableError } from "../catalog/metadata-generation-models.js";
|
||||
import {
|
||||
CatalogOperationInProgressError,
|
||||
CatalogUnavailableError,
|
||||
DescriptionGenerationRunActiveError,
|
||||
type CatalogRepository,
|
||||
type DescriptionGenerationEvent,
|
||||
type DescriptionGenerationRun,
|
||||
} from "../catalog/types.js";
|
||||
|
||||
const idSchema = z.uuid();
|
||||
const modelIdSchema = z.string().regex(/^[a-z][a-z0-9._-]{0,63}$/);
|
||||
const selectedTargetIdsSchema = z.array(idSchema).min(1);
|
||||
const startSchema = z.discriminatedUnion("scope", [
|
||||
z.object({
|
||||
modelId: modelIdSchema,
|
||||
scope: z.literal("selected_columns"),
|
||||
targetIds: selectedTargetIdsSchema,
|
||||
}).strict(),
|
||||
z.object({
|
||||
modelId: modelIdSchema,
|
||||
scope: z.literal("selected_tables"),
|
||||
targetIds: selectedTargetIdsSchema,
|
||||
}).strict(),
|
||||
z.object({ modelId: modelIdSchema, scope: z.literal("all") }).strict(),
|
||||
z.object({ modelId: modelIdSchema, scope: z.literal("missing") }).strict(),
|
||||
]);
|
||||
const eventQuerySchema = z.object({
|
||||
after: z.coerce.number().int().nonnegative().default(0),
|
||||
}).strict();
|
||||
const historyQuerySchema = z.object({
|
||||
limit: z.coerce.number().int().min(1).max(100).default(50),
|
||||
}).strict();
|
||||
const terminalStatuses = new Set<DescriptionGenerationRun["status"]>([
|
||||
"completed",
|
||||
"completed_with_errors",
|
||||
"cancelled",
|
||||
"failed",
|
||||
"interrupted",
|
||||
]);
|
||||
|
||||
function manage(request: FastifyRequest, reply: FastifyReply) {
|
||||
return isPrincipalContext(requirePermission(request, reply, "database.manage"));
|
||||
}
|
||||
|
||||
function publicEvent(event: DescriptionGenerationEvent) {
|
||||
return {
|
||||
sequence: event.sequence,
|
||||
level: event.level,
|
||||
message: event.message,
|
||||
createdAt: event.createdAt,
|
||||
};
|
||||
}
|
||||
|
||||
function publicRun(run: DescriptionGenerationRun) {
|
||||
return {
|
||||
id: run.id,
|
||||
databaseId: run.databaseId,
|
||||
scope: run.scope,
|
||||
modelId: run.modelId,
|
||||
language: run.language,
|
||||
status: run.status,
|
||||
total: run.total,
|
||||
processed: run.processed,
|
||||
generated: run.generated,
|
||||
nonGeneratable: run.nonGeneratable,
|
||||
failed: run.failed,
|
||||
createdAt: run.createdAt,
|
||||
startedAt: run.startedAt,
|
||||
updatedAt: run.updatedAt,
|
||||
finishedAt: run.finishedAt,
|
||||
errorSummary: run.errorSummary,
|
||||
};
|
||||
}
|
||||
|
||||
function safeError(reply: FastifyReply, error: unknown) {
|
||||
if (error instanceof CatalogUnavailableError) {
|
||||
return reply.code(503).send({
|
||||
code: "catalog_unavailable",
|
||||
message: "Database catalog is unavailable.",
|
||||
});
|
||||
}
|
||||
if (error instanceof DescriptionGenerationRunActiveError) {
|
||||
return reply.code(409).send({
|
||||
code: "description_generation_run_active",
|
||||
message: "A Description Generation Run is already active.",
|
||||
});
|
||||
}
|
||||
if (error instanceof DescriptionGenerationRunLiveError) {
|
||||
return reply.code(409).send({
|
||||
code: "description_generation_run_live",
|
||||
message: "A local Description Generation worker or helper is still running.",
|
||||
});
|
||||
}
|
||||
if (error instanceof CatalogOperationInProgressError) {
|
||||
return reply.code(409).send({
|
||||
code: "database_operation_in_progress",
|
||||
message: "A database operation is already in progress.",
|
||||
});
|
||||
}
|
||||
if (error instanceof MetadataGenerationModelUnavailableError) {
|
||||
return reply.code(409).send({
|
||||
code: "metadata_generation_model_unavailable",
|
||||
message: "The selected metadata-generation model is unavailable.",
|
||||
});
|
||||
}
|
||||
if (error instanceof DescriptionGenerationDuplicateTargetIdsError) {
|
||||
return reply.code(400).send({
|
||||
code: "description_generation_target_ids_duplicate",
|
||||
message: "Description generation target IDs must be unique.",
|
||||
});
|
||||
}
|
||||
if (error instanceof DescriptionGenerationTargetIdsRequiredError) {
|
||||
return reply.code(400).send({
|
||||
code: "description_generation_request_invalid",
|
||||
message: "At least one description generation target ID is required.",
|
||||
});
|
||||
}
|
||||
if (error instanceof DescriptionGenerationNoEligibleTargetsError) {
|
||||
return reply.code(409).send({
|
||||
code: "description_generation_no_eligible_targets",
|
||||
message: error.scope === "all"
|
||||
? "No Catalog Tables or Catalog Columns are available for description generation."
|
||||
: "No Catalog Tables or Catalog Columns have a missing Generated Description.",
|
||||
});
|
||||
}
|
||||
if (error instanceof DescriptionGenerationTargetNotFoundError) {
|
||||
const code = error.target === "database"
|
||||
? "database_not_found"
|
||||
: error.target === "table"
|
||||
? "catalog_table_not_found"
|
||||
: "catalog_column_not_found";
|
||||
const message = error.target === "database"
|
||||
? "Database configuration was not found."
|
||||
: error.target === "table"
|
||||
? "One or more selected Catalog Tables were not found."
|
||||
: "One or more selected Catalog Columns were not found.";
|
||||
return reply.code(404).send({
|
||||
code,
|
||||
message,
|
||||
});
|
||||
}
|
||||
if (error instanceof DescriptionGenerationWorkspaceUnavailableError) {
|
||||
return reply.code(409).send({
|
||||
code: "workspace_configuration_unavailable",
|
||||
message: "The database workspace configuration is unavailable.",
|
||||
});
|
||||
}
|
||||
if (error instanceof z.ZodError) {
|
||||
return reply.code(400).send({
|
||||
code: "description_generation_request_invalid",
|
||||
message: "Description generation request is invalid.",
|
||||
});
|
||||
}
|
||||
return reply.code(500).send({
|
||||
code: "description_generation_failed",
|
||||
message: "Description generation failed.",
|
||||
});
|
||||
}
|
||||
|
||||
export function catalogDescriptionGenerationRoutes(
|
||||
app: FastifyInstance,
|
||||
deps: { repository: CatalogRepository; worker: DescriptionGenerationWorker },
|
||||
): void {
|
||||
app.post("/catalog/databases/:databaseId/description-generation-runs", async (request, reply) => {
|
||||
if (!manage(request, reply)) return reply;
|
||||
try {
|
||||
const databaseId = idSchema.parse((request.params as { databaseId?: unknown }).databaseId);
|
||||
const input = startSchema.parse(request.body);
|
||||
const targetIds = "targetIds" in input ? input.targetIds : [];
|
||||
const run = await deps.worker.start(databaseId, input.modelId, input.scope, targetIds);
|
||||
return reply.code(202).send(publicRun(run));
|
||||
} catch (error) {
|
||||
return safeError(reply, error);
|
||||
}
|
||||
});
|
||||
|
||||
app.get("/catalog/description-generation-runs", async (request, reply) => {
|
||||
if (!manage(request, reply)) return reply;
|
||||
try {
|
||||
const { limit } = historyQuerySchema.parse(request.query);
|
||||
return (await deps.repository.listDescriptionGenerationRuns(limit)).map(publicRun);
|
||||
} catch (error) {
|
||||
return safeError(reply, error);
|
||||
}
|
||||
});
|
||||
|
||||
app.get("/catalog/description-generation-runs/:runId", async (request, reply) => {
|
||||
if (!manage(request, reply)) return reply;
|
||||
try {
|
||||
const runId = idSchema.parse((request.params as { runId?: unknown }).runId);
|
||||
const run = await deps.repository.getDescriptionGenerationRun(runId);
|
||||
return run ? publicRun(run) : reply.code(404).send({
|
||||
code: "description_generation_run_not_found",
|
||||
message: "Description Generation Run was not found.",
|
||||
});
|
||||
} catch (error) {
|
||||
return safeError(reply, error);
|
||||
}
|
||||
});
|
||||
|
||||
app.post("/catalog/description-generation-runs/:runId/cancel", async (request, reply) => {
|
||||
if (!manage(request, reply)) return reply;
|
||||
try {
|
||||
const runId = idSchema.parse((request.params as { runId?: unknown }).runId);
|
||||
const run = await deps.worker.cancel(runId);
|
||||
return run ? publicRun(run) : reply.code(404).send({
|
||||
code: "description_generation_run_not_found",
|
||||
message: "Description Generation Run was not found.",
|
||||
});
|
||||
} catch (error) {
|
||||
return safeError(reply, error);
|
||||
}
|
||||
});
|
||||
|
||||
app.post("/catalog/description-generation-runs/unlock", async (request, reply) => {
|
||||
if (!manage(request, reply)) return reply;
|
||||
try {
|
||||
const run = await deps.worker.unlock();
|
||||
return run ? publicRun(run) : reply.code(404).send({
|
||||
code: "description_generation_run_not_found",
|
||||
message: "No stale active Description Generation Run was found.",
|
||||
});
|
||||
} catch (error) {
|
||||
return safeError(reply, error);
|
||||
}
|
||||
});
|
||||
|
||||
app.get("/catalog/description-generation-runs/:runId/events", async (request, reply) => {
|
||||
if (!manage(request, reply)) return reply;
|
||||
let unsubscribe: (() => void) | undefined;
|
||||
let hijacked = false;
|
||||
try {
|
||||
const runId = idSchema.parse((request.params as { runId?: unknown }).runId);
|
||||
let after = eventQuerySchema.parse(request.query).after;
|
||||
const headerCursor = Number(request.headers["last-event-id"]);
|
||||
if (Number.isInteger(headerCursor) && headerCursor >= 0) after = Math.max(after, headerCursor);
|
||||
if (!(await deps.repository.getDescriptionGenerationRun(runId))) {
|
||||
return reply.code(404).send({
|
||||
code: "description_generation_run_not_found",
|
||||
message: "Description Generation Run was not found.",
|
||||
});
|
||||
}
|
||||
|
||||
const buffered: DescriptionGenerationEvent[] = [];
|
||||
let ready = false;
|
||||
let closed = false;
|
||||
let lastRunSnapshot = "";
|
||||
let delivery = Promise.resolve();
|
||||
const close = () => {
|
||||
if (closed) return;
|
||||
closed = true;
|
||||
unsubscribe?.();
|
||||
if (!reply.raw.destroyed) reply.raw.end();
|
||||
};
|
||||
const writeRun = (run: DescriptionGenerationRun) => {
|
||||
if (closed) return;
|
||||
const snapshot = JSON.stringify(publicRun(run));
|
||||
if (snapshot === lastRunSnapshot) return;
|
||||
lastRunSnapshot = snapshot;
|
||||
reply.raw.write(`event: run\ndata: ${snapshot}\n\n`);
|
||||
if (terminalStatuses.has(run.status)) close();
|
||||
};
|
||||
const enqueue = (event: DescriptionGenerationEvent) => {
|
||||
delivery = delivery.then(async () => {
|
||||
if (closed || event.sequence <= after) return;
|
||||
after = event.sequence;
|
||||
reply.raw.write(
|
||||
`id: ${event.sequence}\nevent: log\ndata: ${JSON.stringify(publicEvent(event))}\n\n`,
|
||||
);
|
||||
const run = await deps.repository.getDescriptionGenerationRun(runId);
|
||||
if (run) writeRun(run);
|
||||
}).catch(close);
|
||||
};
|
||||
unsubscribe = deps.worker.subscribeEvents(runId, (event) => {
|
||||
if (ready) enqueue(event);
|
||||
else buffered.push(event);
|
||||
});
|
||||
const persisted = await deps.repository.listDescriptionGenerationEvents(runId, after);
|
||||
|
||||
reply.hijack();
|
||||
hijacked = true;
|
||||
reply.raw.writeHead(200, {
|
||||
"content-type": "text/event-stream; charset=utf-8",
|
||||
"cache-control": "no-cache, no-transform",
|
||||
connection: "keep-alive",
|
||||
"x-accel-buffering": "no",
|
||||
});
|
||||
reply.raw.once("close", close);
|
||||
request.raw.once("aborted", close);
|
||||
for (const event of persisted) {
|
||||
if (event.sequence <= after) continue;
|
||||
after = event.sequence;
|
||||
reply.raw.write(
|
||||
`id: ${event.sequence}\nevent: log\ndata: ${JSON.stringify(publicEvent(event))}\n\n`,
|
||||
);
|
||||
}
|
||||
ready = true;
|
||||
buffered.sort((a, b) => a.sequence - b.sequence).forEach(enqueue);
|
||||
let pending = delivery;
|
||||
await pending;
|
||||
while (pending !== delivery) {
|
||||
pending = delivery;
|
||||
await pending;
|
||||
}
|
||||
const current = await deps.repository.getDescriptionGenerationRun(runId);
|
||||
if (current) writeRun(current);
|
||||
return reply;
|
||||
} catch (error) {
|
||||
unsubscribe?.();
|
||||
if (hijacked) {
|
||||
if (!reply.raw.destroyed) reply.raw.end();
|
||||
return reply;
|
||||
}
|
||||
return safeError(reply, error);
|
||||
}
|
||||
});
|
||||
|
||||
app.get("/catalog/description-generation-runs/:runId/events-list", async (request, reply) => {
|
||||
if (!manage(request, reply)) return reply;
|
||||
try {
|
||||
const runId = idSchema.parse((request.params as { runId?: unknown }).runId);
|
||||
const { after } = eventQuerySchema.parse(request.query);
|
||||
if (!(await deps.repository.getDescriptionGenerationRun(runId))) {
|
||||
return reply.code(404).send({
|
||||
code: "description_generation_run_not_found",
|
||||
message: "Description Generation Run was not found.",
|
||||
});
|
||||
}
|
||||
return (await deps.repository.listDescriptionGenerationEvents(runId, after)).map(publicEvent);
|
||||
} catch (error) {
|
||||
return safeError(reply, error);
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -43,7 +43,13 @@ function safeError(reply: FastifyReply, error: unknown) {
|
||||
if (error instanceof CatalogUnavailableError) {
|
||||
return reply.code(503).send({ code: "catalog_unavailable", message: "Database catalog is unavailable." });
|
||||
}
|
||||
if (error instanceof CatalogConflictError || error instanceof CatalogOperationInProgressError) {
|
||||
if (error instanceof CatalogOperationInProgressError) {
|
||||
return reply.code(409).send({
|
||||
code: "database_operation_in_progress",
|
||||
message: "A database operation is already in progress.",
|
||||
});
|
||||
}
|
||||
if (error instanceof CatalogConflictError) {
|
||||
return reply.code(409).send({ code: "schema_sync_conflict", message: error.message });
|
||||
}
|
||||
if (error instanceof CatalogConnectorError) {
|
||||
@@ -98,21 +104,24 @@ export function catalogSchemaRoutes(
|
||||
const tableId = idSchema.parse(params.tableId);
|
||||
const columnId = idSchema.parse(params.columnId);
|
||||
const input = metadataSchema.parse(request.body);
|
||||
const current = await deps.repository.getColumn(databaseId, tableId, columnId);
|
||||
if (!current) return reply.code(404).send({ code: "column_not_found", message: "Catalog column was not found." });
|
||||
if (current.version !== input.version) {
|
||||
return reply.code(409).send({ code: "column_stale", message: "Column metadata changed. Reload and try again." });
|
||||
}
|
||||
const updated = await deps.repository.updateColumnMetadata(
|
||||
databaseId,
|
||||
tableId,
|
||||
columnId,
|
||||
input.version,
|
||||
normalized(input.description),
|
||||
normalized(input.generatedDescription),
|
||||
);
|
||||
if (!updated) return reply.code(409).send({ code: "column_stale", message: "Column metadata changed. Reload and try again." });
|
||||
return updated;
|
||||
const operation = async () => {
|
||||
const current = await deps.repository.getColumn(databaseId, tableId, columnId);
|
||||
if (!current) return reply.code(404).send({ code: "column_not_found", message: "Catalog column was not found." });
|
||||
if (current.version !== input.version) {
|
||||
return reply.code(409).send({ code: "column_stale", message: "Column metadata changed. Reload and try again." });
|
||||
}
|
||||
const updated = await deps.repository.updateColumnMetadata(
|
||||
databaseId,
|
||||
tableId,
|
||||
columnId,
|
||||
input.version,
|
||||
normalized(input.description),
|
||||
normalized(input.generatedDescription),
|
||||
);
|
||||
if (!updated) return reply.code(409).send({ code: "column_stale", message: "Column metadata changed. Reload and try again." });
|
||||
return updated;
|
||||
};
|
||||
return deps.operations ? await deps.operations.run(databaseId, operation) : await operation();
|
||||
} catch (error) { return safeError(reply, error); }
|
||||
});
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@ import type { FastifyInstance, FastifyReply, FastifyRequest } from "fastify";
|
||||
import { z } from "zod";
|
||||
import { isPrincipalContext, requirePermission } from "../auth/authorization.js";
|
||||
import type { CatalogTableService } from "../catalog/table-service.js";
|
||||
import type { CatalogOperationCoordinator } from "../catalog/operation-coordinator.js";
|
||||
import {
|
||||
CatalogConnectorError,
|
||||
CatalogOperationInProgressError,
|
||||
@@ -38,7 +39,11 @@ function safeError(reply: FastifyReply, error: unknown) {
|
||||
|
||||
export function catalogTableRoutes(
|
||||
app: FastifyInstance,
|
||||
deps: { repository: CatalogRepository; service: CatalogTableService },
|
||||
deps: {
|
||||
repository: CatalogRepository;
|
||||
service: CatalogTableService;
|
||||
operations: CatalogOperationCoordinator;
|
||||
},
|
||||
): void {
|
||||
app.get("/catalog/databases/:databaseId/tables", async (request, reply) => {
|
||||
if (!manage(request, reply)) return reply;
|
||||
@@ -59,20 +64,22 @@ export function catalogTableRoutes(
|
||||
const tableId = idSchema.parse(params.tableId);
|
||||
const input = updateSchema.parse(request.body);
|
||||
const { version, description } = input;
|
||||
const current = await deps.repository.getTable(databaseId, tableId);
|
||||
if (!current) return reply.code(404).send({ code: "table_not_found", message: "Catalog table was not found." });
|
||||
if (current.version !== version) {
|
||||
return reply.code(409).send({ code: "table_stale", message: "Table description changed. Reload and try again." });
|
||||
}
|
||||
const updated = await deps.service.updateMetadata(
|
||||
databaseId,
|
||||
tableId,
|
||||
version,
|
||||
description,
|
||||
input.generatedDescription === undefined ? current.generatedDescription : input.generatedDescription,
|
||||
);
|
||||
if (!updated) return reply.code(409).send({ code: "table_stale", message: "Table description changed. Reload and try again." });
|
||||
return updated;
|
||||
return await deps.operations.run(databaseId, async () => {
|
||||
const current = await deps.repository.getTable(databaseId, tableId);
|
||||
if (!current) return reply.code(404).send({ code: "table_not_found", message: "Catalog table was not found." });
|
||||
if (current.version !== version) {
|
||||
return reply.code(409).send({ code: "table_stale", message: "Table description changed. Reload and try again." });
|
||||
}
|
||||
const updated = await deps.service.updateMetadata(
|
||||
databaseId,
|
||||
tableId,
|
||||
version,
|
||||
description,
|
||||
input.generatedDescription === undefined ? current.generatedDescription : input.generatedDescription,
|
||||
);
|
||||
if (!updated) return reply.code(409).send({ code: "table_stale", message: "Table description changed. Reload and try again." });
|
||||
return updated;
|
||||
});
|
||||
} catch (error) { return safeError(reply, error); }
|
||||
});
|
||||
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
import type { FastifyInstance } from "fastify";
|
||||
import { isPrincipalContext, requirePermission } from "../auth/authorization.js";
|
||||
import type { MetadataGenerationModels } from "../catalog/metadata-generation-models.js";
|
||||
|
||||
export function metadataGenerationModelRoutes(
|
||||
app: FastifyInstance,
|
||||
models: MetadataGenerationModels,
|
||||
): void {
|
||||
app.get("/catalog/metadata-generation/models", async (request, reply) => {
|
||||
if (!isPrincipalContext(requirePermission(request, reply, "database.manage"))) return reply;
|
||||
return models.catalog();
|
||||
});
|
||||
}
|
||||
Reference in New Issue
Block a user