feat: implement metadata catalog database management
This commit is contained in:
Codex
Generated
+2049
File diff suppressed because it is too large
Load Diff
@@ -6,6 +6,7 @@
|
||||
"dev": "tsx watch src/server.ts",
|
||||
"prebuild": "node scripts/clean-dist.mjs",
|
||||
"build": "tsc -p tsconfig.json",
|
||||
"catalog:migrate": "node dist/catalog/migrate.js",
|
||||
"test": "vitest run",
|
||||
"start": "node dist/server.js",
|
||||
"test:schema-v3-verifier": "python3 -I -B scripts/test_revision_state_policy.py && node --test scripts/verify-workspace-descriptor-files.test.mjs scripts/revision-state-policy.test.mjs"
|
||||
@@ -16,12 +17,14 @@
|
||||
"@fastify/rate-limit": "11.2.0",
|
||||
"@types/pg": "^8.20.3",
|
||||
"fastify": "^5.0.0",
|
||||
"kysely": "^0.29.5",
|
||||
"openid-client": "6.8.5",
|
||||
"pg": "^8.22.0",
|
||||
"yaml": "^2.9.0",
|
||||
"zod": "^4.4.3"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@testcontainers/postgresql": "^12.1.0",
|
||||
"@types/node": "24.13.3",
|
||||
"tsx": "^4.19.0",
|
||||
"typescript": "^5.6.0",
|
||||
|
||||
Executable
+6
@@ -0,0 +1,6 @@
|
||||
#!/usr/bin/env node
|
||||
import { readFileSync } from "node:fs";
|
||||
|
||||
const path = process.env.THT_SSH_PASSPHRASE_FILE;
|
||||
if (!path) process.exit(1);
|
||||
process.stdout.write(readFileSync(path));
|
||||
@@ -37,6 +37,17 @@ import { supportsSessionRuntime } from "./workspaces/bindings.js";
|
||||
import { resolveRuntimeBindingsWithWorkspaceSecrets } from "./workspaces/secret-requirements.js";
|
||||
import type { WorkspaceDescriptor } from "./workspaces/schema.js";
|
||||
import { WorkspaceSecretStore } from "./workspaces/secret-store.js";
|
||||
import { createCatalogRepository } from "./catalog/repository.js";
|
||||
import type { CatalogRepository } from "./catalog/types.js";
|
||||
import { CatalogService } from "./catalog/service.js";
|
||||
import { catalogDatabaseRoutes } from "./routes/catalog-databases.js";
|
||||
import { CatalogOperationCoordinator } from "./catalog/operation-coordinator.js";
|
||||
import { ConcreteCatalogPostgresAccess, type CatalogPostgresAccess } from "./catalog/postgres-access.js";
|
||||
import { CatalogTableService } from "./catalog/table-service.js";
|
||||
import { catalogTableRoutes } from "./routes/catalog-tables.js";
|
||||
import { ConcreteCatalogSchemaIntrospector, type CatalogSchemaIntrospector } from "./catalog/schema-introspector.js";
|
||||
import { CatalogSyncWorker } from "./catalog/sync-worker.js";
|
||||
import { catalogSchemaRoutes } from "./routes/catalog-schema.js";
|
||||
|
||||
export interface BuildAppDeps {
|
||||
thtRunner?: ThtRunner;
|
||||
@@ -49,6 +60,13 @@ export interface BuildAppDeps {
|
||||
workspaceRegistry?: WorkspaceRegistry;
|
||||
workspaceDiagnoser?: WorkspaceDiagnoser;
|
||||
workspaceSecretStore?: WorkspaceSecretStore;
|
||||
catalogRepository?: CatalogRepository;
|
||||
catalogService?: CatalogService;
|
||||
catalogPostgresAccess?: CatalogPostgresAccess;
|
||||
catalogTableService?: CatalogTableService;
|
||||
catalogSchemaIntrospector?: CatalogSchemaIntrospector;
|
||||
catalogSyncWorker?: CatalogSyncWorker;
|
||||
catalogOperationCoordinator?: CatalogOperationCoordinator;
|
||||
workspaceRuntimeSupport?: (workspace: WorkspaceDescriptor) => boolean;
|
||||
maintenanceBarrier?: MaintenanceBarrier;
|
||||
piManagement?: PiManagementService;
|
||||
@@ -121,6 +139,37 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
|
||||
const mgr = deps?.mgr ?? new PiProcessManager(config, deps?.spawnFn ? { spawnFn: deps.spawnFn } : undefined);
|
||||
const hub = deps?.hub ?? new SseHub();
|
||||
const workspaceRegistry = deps?.workspaceRegistry ?? new WorkspaceRegistry(config.workspaceRegistry);
|
||||
const catalogRepository = deps?.catalogRepository ?? createCatalogRepository(config.catalogDatabase);
|
||||
const catalogOperationCoordinator = deps?.catalogOperationCoordinator ?? new CatalogOperationCoordinator();
|
||||
const catalogPostgresAccess = deps?.catalogPostgresAccess ?? new ConcreteCatalogPostgresAccess(
|
||||
workspaceSecretStore,
|
||||
{ connectTimeoutMs: config.workspaceDiagnosticTimeoutMs },
|
||||
);
|
||||
const catalogService = deps?.catalogService ?? new CatalogService(
|
||||
catalogRepository,
|
||||
workspaceRegistry,
|
||||
workspaceSecretStore,
|
||||
config.workspaceRegistry.secretRoots,
|
||||
config.workspaceDiagnosticTimeoutMs,
|
||||
catalogPostgresAccess,
|
||||
catalogOperationCoordinator,
|
||||
);
|
||||
const catalogTableService = deps?.catalogTableService ?? new CatalogTableService(catalogRepository);
|
||||
const catalogSchemaIntrospector = deps?.catalogSchemaIntrospector ?? new ConcreteCatalogSchemaIntrospector(
|
||||
catalogPostgresAccess,
|
||||
workspaceSecretStore,
|
||||
);
|
||||
const catalogSyncWorker = deps?.catalogSyncWorker ?? new CatalogSyncWorker(
|
||||
catalogRepository,
|
||||
catalogSchemaIntrospector,
|
||||
catalogOperationCoordinator,
|
||||
config.catalogSyncTimeoutMs,
|
||||
);
|
||||
app.addHook("onReady", async () => { await catalogSyncWorker.initialize(); });
|
||||
if (!deps?.catalogRepository && catalogRepository.close) {
|
||||
app.addHook("onClose", async () => { await catalogRepository.close?.(); });
|
||||
}
|
||||
app.addHook("onClose", async () => { await catalogSyncWorker.stop(); });
|
||||
const workspaceDiagnoser = deps?.workspaceDiagnoser
|
||||
?? createProductionWorkspaceDiagnoser(config.workspaceDiagnosticTimeoutMs, undefined, {
|
||||
internalQdrantUrl: config.internalQdrantUrl,
|
||||
@@ -334,6 +383,9 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
|
||||
authDiagnoser,
|
||||
secretStore: workspaceSecretStore,
|
||||
});
|
||||
catalogDatabaseRoutes(app, { repository: catalogRepository, service: catalogService, operations: catalogOperationCoordinator });
|
||||
catalogTableRoutes(app, { repository: catalogRepository, service: catalogTableService });
|
||||
catalogSchemaRoutes(app, { repository: catalogRepository, worker: catalogSyncWorker });
|
||||
settingsRoutes(app, { cfg: config, listModels, getSettings });
|
||||
piManagementRoutes(app, { service: piManagement });
|
||||
|
||||
|
||||
@@ -38,7 +38,7 @@ const MAX_MAPPED_GROUPS = 128;
|
||||
const ROLES = ["user", "admin"] as const;
|
||||
export const PERMISSION_CATALOG: readonly Permission[] = [
|
||||
"session.use", "session.read_all", "session.manage_all", "settings.manage",
|
||||
"workspace.manage", "workspace.secrets.manage", "pi.manage", "auth.diagnostics.read",
|
||||
"workspace.manage", "workspace.secrets.manage", "database.manage", "pi.manage", "auth.diagnostics.read",
|
||||
];
|
||||
|
||||
const invalid = (): Error => new Error("authentication configuration is invalid");
|
||||
|
||||
@@ -33,7 +33,7 @@ const EMPTY_HKDF_SALT = Buffer.alloc(0);
|
||||
const ROLES = ["user", "admin"] as const;
|
||||
const PERMISSIONS = [
|
||||
"session.use", "session.read_all", "session.manage_all", "settings.manage",
|
||||
"workspace.manage", "workspace.secrets.manage", "pi.manage", "auth.diagnostics.read",
|
||||
"workspace.manage", "workspace.secrets.manage", "database.manage", "pi.manage", "auth.diagnostics.read",
|
||||
] as const satisfies readonly Permission[];
|
||||
|
||||
const invalid = (): Error => new Error("auth_session_store_invalid");
|
||||
|
||||
@@ -5,7 +5,7 @@ export type Role = "user" | "admin";
|
||||
export type Permission =
|
||||
| "session.use" | "session.read_all" | "session.manage_all"
|
||||
| "settings.manage" | "workspace.manage" | "workspace.secrets.manage"
|
||||
| "pi.manage" | "auth.diagnostics.read";
|
||||
| "database.manage" | "pi.manage" | "auth.diagnostics.read";
|
||||
|
||||
export interface AuthenticationSessionConfig {
|
||||
regularTtlSeconds: number;
|
||||
|
||||
@@ -0,0 +1,692 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import {
|
||||
CatalogConflictError,
|
||||
CatalogConnectorError,
|
||||
type CatalogColumn,
|
||||
type CatalogRelationship,
|
||||
type CatalogSchemaDiff,
|
||||
type CatalogSyncCounts,
|
||||
type CatalogSyncEvent,
|
||||
type CatalogSyncRun,
|
||||
type CatalogSyncRunUpdate,
|
||||
type CatalogSyncScope,
|
||||
type CatalogTable,
|
||||
type CatalogRepository,
|
||||
type DatabaseConfigurationInput,
|
||||
type DatabaseTestResult,
|
||||
type ObservedCatalogTable,
|
||||
type ObservedSchemaSnapshot,
|
||||
type TableSyncRepositoryResult,
|
||||
type WorkspaceDatabase,
|
||||
} from "./types.js";
|
||||
|
||||
function clone(value: WorkspaceDatabase): WorkspaceDatabase {
|
||||
return structuredClone(value);
|
||||
}
|
||||
|
||||
/** Deterministic repository used by route tests and local contract consumers. */
|
||||
export class MemoryCatalogRepository implements CatalogRepository {
|
||||
private readonly records = new Map<string, WorkspaceDatabase>();
|
||||
private readonly tables = new Map<string, CatalogTable>();
|
||||
private readonly columns = new Map<string, CatalogColumn>();
|
||||
private readonly relationships = new Map<string, CatalogRelationship>();
|
||||
private readonly syncRuns = new Map<string, CatalogSyncRun>();
|
||||
private readonly syncEvents = new Map<string, CatalogSyncEvent[]>();
|
||||
|
||||
async list(): Promise<WorkspaceDatabase[]> {
|
||||
return [...this.records.values()].sort((a, b) => a.workspaceId.localeCompare(b.workspaceId)).map(clone);
|
||||
}
|
||||
async get(id: string): Promise<WorkspaceDatabase | undefined> {
|
||||
const value = this.records.get(id);
|
||||
return value ? clone(value) : undefined;
|
||||
}
|
||||
async getByWorkspace(workspaceId: string): Promise<WorkspaceDatabase | undefined> {
|
||||
const value = [...this.records.values()].find((record) => record.workspaceId === workspaceId);
|
||||
return value ? clone(value) : undefined;
|
||||
}
|
||||
async create(input: DatabaseConfigurationInput): Promise<WorkspaceDatabase> {
|
||||
if ([...this.records.values()].some((record) => record.workspaceId === input.workspaceId)) {
|
||||
throw new CatalogConflictError("Workspace database already exists");
|
||||
}
|
||||
const now = new Date().toISOString();
|
||||
const record: WorkspaceDatabase = {
|
||||
id: randomUUID(),
|
||||
...structuredClone(input),
|
||||
version: 1,
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
connectionStatus: "untested",
|
||||
};
|
||||
this.records.set(record.id, record);
|
||||
return clone(record);
|
||||
}
|
||||
async update(id: string, expectedVersion: number, input: DatabaseConfigurationInput): Promise<WorkspaceDatabase | undefined> {
|
||||
const current = this.records.get(id);
|
||||
if (!current || current.version !== expectedVersion) return undefined;
|
||||
if ([...this.records.values()].some((record) => record.id !== id && record.workspaceId === input.workspaceId)) {
|
||||
throw new CatalogConflictError("Workspace database already exists");
|
||||
}
|
||||
const updated: WorkspaceDatabase = {
|
||||
...current,
|
||||
...structuredClone(input),
|
||||
version: current.version + 1,
|
||||
updatedAt: new Date().toISOString(),
|
||||
connectionStatus: "untested",
|
||||
testedVersion: undefined,
|
||||
lastTestedAt: undefined,
|
||||
lastErrorCode: undefined,
|
||||
lastErrorMessage: undefined,
|
||||
schemaSyncedVersion: undefined,
|
||||
schemaSyncedAt: undefined,
|
||||
};
|
||||
this.records.set(id, updated);
|
||||
return clone(updated);
|
||||
}
|
||||
async recordTest(id: string, expectedVersion: number, result: DatabaseTestResult): Promise<WorkspaceDatabase | undefined> {
|
||||
const current = this.records.get(id);
|
||||
if (!current || current.version !== expectedVersion) return undefined;
|
||||
const updated = {
|
||||
...current,
|
||||
connectionStatus: result.connectionStatus,
|
||||
testedVersion: result.testedVersion,
|
||||
lastTestedAt: result.lastTestedAt,
|
||||
lastErrorCode: result.errorCode,
|
||||
lastErrorMessage: result.errorMessage,
|
||||
};
|
||||
this.records.set(id, updated);
|
||||
return clone(updated);
|
||||
}
|
||||
async touch(id: string, expectedVersion: number): Promise<WorkspaceDatabase | undefined> {
|
||||
const current = this.records.get(id);
|
||||
if (!current || current.version !== expectedVersion) return undefined;
|
||||
return await this.update(id, expectedVersion, {
|
||||
workspaceId: current.workspaceId,
|
||||
engine: current.engine,
|
||||
databaseName: current.databaseName,
|
||||
schema: current.schema,
|
||||
binding: current.binding,
|
||||
});
|
||||
}
|
||||
async delete(id: string, expectedVersion: number): Promise<boolean> {
|
||||
const current = this.records.get(id);
|
||||
if (!current || current.version !== expectedVersion) return false;
|
||||
for (const [tableId, table] of this.tables) {
|
||||
if (table.databaseId === id) {
|
||||
this.tables.delete(tableId);
|
||||
for (const [columnId, column] of this.columns) if (column.tableId === tableId) this.columns.delete(columnId);
|
||||
}
|
||||
}
|
||||
for (const [relationshipId, relationship] of this.relationships) {
|
||||
if (relationship.databaseId === id) this.relationships.delete(relationshipId);
|
||||
}
|
||||
return this.records.delete(id);
|
||||
}
|
||||
async listTables(databaseId: string): Promise<CatalogTable[]> {
|
||||
return [...this.tables.values()]
|
||||
.filter((table) => table.databaseId === databaseId)
|
||||
.sort((a, b) => a.name.localeCompare(b.name))
|
||||
.map((table) => structuredClone(table));
|
||||
}
|
||||
async getTable(databaseId: string, tableId: string): Promise<CatalogTable | undefined> {
|
||||
const table = this.tables.get(tableId);
|
||||
return table?.databaseId === databaseId ? structuredClone(table) : undefined;
|
||||
}
|
||||
async updateTableDescription(
|
||||
databaseId: string,
|
||||
tableId: string,
|
||||
expectedVersion: number,
|
||||
description: string | null,
|
||||
): Promise<CatalogTable | undefined> {
|
||||
const current = this.tables.get(tableId);
|
||||
if (!current || current.databaseId !== databaseId || current.version !== expectedVersion) return undefined;
|
||||
const updated = {
|
||||
...current,
|
||||
description,
|
||||
version: current.version + 1,
|
||||
updatedAt: new Date().toISOString(),
|
||||
};
|
||||
this.tables.set(tableId, updated);
|
||||
return structuredClone(updated);
|
||||
}
|
||||
async updateTableMetadata(
|
||||
databaseId: string,
|
||||
tableId: string,
|
||||
expectedVersion: number,
|
||||
description: string | null,
|
||||
generatedDescription: string | null,
|
||||
): Promise<CatalogTable | undefined> {
|
||||
const current = this.tables.get(tableId);
|
||||
if (!current || current.databaseId !== databaseId || current.version !== expectedVersion) return undefined;
|
||||
const updated = {
|
||||
...current, description, generatedDescription, version: current.version + 1,
|
||||
updatedAt: new Date().toISOString(),
|
||||
};
|
||||
this.tables.set(tableId, updated);
|
||||
return structuredClone(updated);
|
||||
}
|
||||
|
||||
async listColumns(databaseId: string, tableId: string): Promise<CatalogColumn[]> {
|
||||
const table = this.tables.get(tableId);
|
||||
if (!table || table.databaseId !== databaseId) return [];
|
||||
return [...this.columns.values()].filter((column) => column.tableId === tableId)
|
||||
.sort((a, b) => a.ordinalPosition - b.ordinalPosition).map((column) => structuredClone(column));
|
||||
}
|
||||
|
||||
async getColumn(databaseId: string, tableId: string, columnId: string): Promise<CatalogColumn | undefined> {
|
||||
const table = this.tables.get(tableId);
|
||||
const column = this.columns.get(columnId);
|
||||
return table?.databaseId === databaseId && column?.tableId === tableId ? structuredClone(column) : undefined;
|
||||
}
|
||||
|
||||
async updateColumnMetadata(
|
||||
databaseId: string,
|
||||
tableId: string,
|
||||
columnId: string,
|
||||
expectedVersion: number,
|
||||
description: string | null,
|
||||
generatedDescription: string | null,
|
||||
): Promise<CatalogColumn | undefined> {
|
||||
const current = await this.getColumn(databaseId, tableId, columnId);
|
||||
if (!current || current.version !== expectedVersion) return undefined;
|
||||
const updated = { ...current, description, generatedDescription, version: current.version + 1, updatedAt: new Date().toISOString() };
|
||||
this.columns.set(columnId, updated);
|
||||
return structuredClone(updated);
|
||||
}
|
||||
|
||||
async listRelationships(databaseId: string): Promise<CatalogRelationship[]> {
|
||||
return [...this.relationships.values()].filter((relationship) => relationship.databaseId === databaseId)
|
||||
.sort((a, b) => `${a.sourceTableName}.${a.constraintName}`.localeCompare(`${b.sourceTableName}.${b.constraintName}`))
|
||||
.map((relationship) => structuredClone(relationship));
|
||||
}
|
||||
|
||||
async planSchemaSync(
|
||||
databaseId: string,
|
||||
scope: CatalogSyncScope,
|
||||
tableIds: readonly string[],
|
||||
snapshot: ObservedSchemaSnapshot,
|
||||
): Promise<CatalogSchemaDiff> {
|
||||
this.assertSnapshotCapability(scope, snapshot);
|
||||
const tables = await this.listTables(databaseId);
|
||||
const selectedTableIds = new Set(tableIds);
|
||||
const selectedTables = scope === "columns" && selectedTableIds.size > 0
|
||||
? tables.filter((table) => selectedTableIds.has(table.id))
|
||||
: tables;
|
||||
const observedTables = new Set(snapshot.tables.map((table) => table.name));
|
||||
const observedColumns = new Set(snapshot.columns.map((column) => `${column.tableName}\u0000${column.name}`));
|
||||
const observedRelationships = new Set(
|
||||
snapshot.relationships.map((relationship) => `${relationship.sourceTableName}\u0000${relationship.constraintName}`),
|
||||
);
|
||||
const deletedTableIds = new Set(tables.filter((table) => !observedTables.has(table.name)).map((table) => table.id));
|
||||
|
||||
return {
|
||||
deletedTables: scope === "tables" || scope === "all"
|
||||
? tables.filter((table) => !observedTables.has(table.name)).map((table) => table.name).sort()
|
||||
: [],
|
||||
deletedColumns: scope === "tables" || scope === "columns" || scope === "all"
|
||||
? [...this.columns.values()]
|
||||
.filter((column) => scope === "tables" ? deletedTableIds.has(column.tableId) : selectedTables.some((table) => table.id === column.tableId))
|
||||
.filter((column) => {
|
||||
const table = tables.find((candidate) => candidate.id === column.tableId);
|
||||
return table && (scope === "tables" || !observedColumns.has(`${table.name}\u0000${column.name}`));
|
||||
})
|
||||
.map((column) => ({
|
||||
tableName: tables.find((table) => table.id === column.tableId)?.name ?? "",
|
||||
columnName: column.name,
|
||||
}))
|
||||
.sort((a, b) => `${a.tableName}.${a.columnName}`.localeCompare(`${b.tableName}.${b.columnName}`))
|
||||
: [],
|
||||
deletedRelationships: scope === "tables" || scope === "relationships" || scope === "all"
|
||||
? [...this.relationships.values()]
|
||||
.filter((relationship) => relationship.databaseId === databaseId)
|
||||
.filter((relationship) => scope === "tables"
|
||||
? deletedTableIds.has(relationship.sourceTableId) || deletedTableIds.has(relationship.targetTableId)
|
||||
: !observedRelationships.has(`${relationship.sourceTableName}\u0000${relationship.constraintName}`))
|
||||
.map((relationship) => ({
|
||||
sourceTableName: relationship.sourceTableName,
|
||||
constraintName: relationship.constraintName,
|
||||
}))
|
||||
.sort((a, b) => `${a.sourceTableName}.${a.constraintName}`.localeCompare(`${b.sourceTableName}.${b.constraintName}`))
|
||||
: [],
|
||||
};
|
||||
}
|
||||
|
||||
async applySchemaSync(
|
||||
databaseId: string,
|
||||
expectedDatabaseVersion: number,
|
||||
scope: CatalogSyncScope,
|
||||
tableIds: readonly string[],
|
||||
snapshot: ObservedSchemaSnapshot,
|
||||
): Promise<CatalogSyncCounts | undefined> {
|
||||
const database = this.records.get(databaseId);
|
||||
if (!database || database.version !== expectedDatabaseVersion) return undefined;
|
||||
this.assertSnapshotCapability(scope, snapshot);
|
||||
|
||||
const tableBackup = structuredClone([...this.tables.entries()]);
|
||||
const columnBackup = structuredClone([...this.columns.entries()]);
|
||||
const relationshipBackup = structuredClone([...this.relationships.entries()]);
|
||||
const now = new Date().toISOString();
|
||||
let created = 0;
|
||||
let updated = 0;
|
||||
let deleted = 0;
|
||||
|
||||
try {
|
||||
if (scope === "tables" || scope === "all") {
|
||||
const observedByName = new Map(snapshot.tables.map((table) => [table.name, table]));
|
||||
const existing = await this.listTables(databaseId);
|
||||
for (const table of existing) {
|
||||
const observed = observedByName.get(table.name);
|
||||
if (!observed) {
|
||||
this.deleteTable(table.id);
|
||||
deleted += 1;
|
||||
} else {
|
||||
const changed = table.sourceComment !== observed.sourceComment;
|
||||
this.tables.set(table.id, {
|
||||
...table,
|
||||
sourceComment: observed.sourceComment,
|
||||
lastSyncedDatabaseVersion: expectedDatabaseVersion,
|
||||
lastSyncedAt: now,
|
||||
version: changed ? table.version + 1 : table.version,
|
||||
updatedAt: changed ? now : table.updatedAt,
|
||||
});
|
||||
if (changed) updated += 1;
|
||||
observedByName.delete(table.name);
|
||||
}
|
||||
}
|
||||
for (const observed of observedByName.values()) {
|
||||
const id = randomUUID();
|
||||
this.tables.set(id, {
|
||||
id,
|
||||
databaseId,
|
||||
name: observed.name,
|
||||
sourceComment: observed.sourceComment,
|
||||
description: null,
|
||||
generatedDescription: null,
|
||||
lastSyncedDatabaseVersion: expectedDatabaseVersion,
|
||||
lastSyncedAt: now,
|
||||
version: 1,
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
});
|
||||
created += 1;
|
||||
}
|
||||
}
|
||||
|
||||
if (scope === "columns" || scope === "all") {
|
||||
const currentTables = await this.listTables(databaseId);
|
||||
const selectedIds = scope === "all" || tableIds.length === 0
|
||||
? new Set(currentTables.map((table) => table.id))
|
||||
: new Set(tableIds);
|
||||
const selectedTables = currentTables.filter((table) => selectedIds.has(table.id));
|
||||
if (scope === "columns" && selectedTables.length !== selectedIds.size) {
|
||||
throw new CatalogConnectorError("One or more selected tables no longer exist");
|
||||
}
|
||||
const selectedNames = new Set(selectedTables.map((table) => table.name));
|
||||
const tableByName = new Map(selectedTables.map((table) => [table.name, table]));
|
||||
const observedByKey = new Map(
|
||||
snapshot.columns
|
||||
.filter((column) => selectedNames.has(column.tableName))
|
||||
.map((column) => [`${column.tableName}\u0000${column.name}`, column]),
|
||||
);
|
||||
for (const column of [...this.columns.values()].filter((candidate) => selectedIds.has(candidate.tableId))) {
|
||||
const table = selectedTables.find((candidate) => candidate.id === column.tableId);
|
||||
if (!table) continue;
|
||||
const key = `${table.name}\u0000${column.name}`;
|
||||
const observed = observedByKey.get(key);
|
||||
if (!observed) {
|
||||
this.deleteColumn(column.id);
|
||||
deleted += 1;
|
||||
} else {
|
||||
const changed = column.ordinalPosition !== observed.ordinalPosition
|
||||
|| column.dataType !== observed.dataType
|
||||
|| column.isNullable !== observed.isNullable
|
||||
|| column.defaultExpression !== observed.defaultExpression
|
||||
|| column.primaryKeyPosition !== observed.primaryKeyPosition
|
||||
|| column.sourceComment !== observed.sourceComment;
|
||||
this.columns.set(column.id, {
|
||||
...column,
|
||||
ordinalPosition: observed.ordinalPosition,
|
||||
dataType: observed.dataType,
|
||||
isNullable: observed.isNullable,
|
||||
defaultExpression: observed.defaultExpression,
|
||||
primaryKeyPosition: observed.primaryKeyPosition,
|
||||
isPrimaryKey: observed.primaryKeyPosition !== null,
|
||||
sourceComment: observed.sourceComment,
|
||||
lastSyncedDatabaseVersion: expectedDatabaseVersion,
|
||||
lastSyncedAt: now,
|
||||
version: changed ? column.version + 1 : column.version,
|
||||
updatedAt: changed ? now : column.updatedAt,
|
||||
});
|
||||
if (changed) updated += 1;
|
||||
observedByKey.delete(key);
|
||||
}
|
||||
}
|
||||
for (const observed of observedByKey.values()) {
|
||||
const table = tableByName.get(observed.tableName);
|
||||
if (!table) continue;
|
||||
const id = randomUUID();
|
||||
this.columns.set(id, {
|
||||
id,
|
||||
tableId: table.id,
|
||||
name: observed.name,
|
||||
ordinalPosition: observed.ordinalPosition,
|
||||
dataType: observed.dataType,
|
||||
isNullable: observed.isNullable,
|
||||
defaultExpression: observed.defaultExpression,
|
||||
primaryKeyPosition: observed.primaryKeyPosition,
|
||||
isPrimaryKey: observed.primaryKeyPosition !== null,
|
||||
isForeignKey: false,
|
||||
foreignKeyCount: 0,
|
||||
sourceComment: observed.sourceComment,
|
||||
description: null,
|
||||
generatedDescription: null,
|
||||
lastSyncedDatabaseVersion: expectedDatabaseVersion,
|
||||
lastSyncedAt: now,
|
||||
version: 1,
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
});
|
||||
created += 1;
|
||||
}
|
||||
}
|
||||
|
||||
if (scope === "relationships" || scope === "all") {
|
||||
const tables = await this.listTables(databaseId);
|
||||
const tableByName = new Map(tables.map((table) => [table.name, table]));
|
||||
const existing = [...this.relationships.values()].filter((relationship) => relationship.databaseId === databaseId);
|
||||
const existingByKey = new Map(existing.map((relationship) => [`${relationship.sourceTableName}\u0000${relationship.constraintName}`, relationship]));
|
||||
const observedKeys = new Set(snapshot.relationships.map((relationship) => `${relationship.sourceTableName}\u0000${relationship.constraintName}`));
|
||||
for (const relationship of existing) {
|
||||
if (!observedKeys.has(`${relationship.sourceTableName}\u0000${relationship.constraintName}`)) {
|
||||
this.relationships.delete(relationship.id);
|
||||
deleted += 1;
|
||||
}
|
||||
}
|
||||
for (const observed of snapshot.relationships) {
|
||||
const sourceTable = tableByName.get(observed.sourceTableName);
|
||||
const targetTable = tableByName.get(observed.targetTableName);
|
||||
if (!sourceTable || !targetTable) {
|
||||
throw new CatalogConnectorError(`Relationship ${observed.constraintName} refers to an unknown table`);
|
||||
}
|
||||
const pairs = observed.columns.map((pair) => {
|
||||
const source = [...this.columns.values()].find((column) => column.tableId === sourceTable.id && column.name === pair.sourceColumnName);
|
||||
const target = [...this.columns.values()].find((column) => column.tableId === targetTable.id && column.name === pair.targetColumnName);
|
||||
if (!source || !target) {
|
||||
throw new CatalogConnectorError(`Relationship ${observed.constraintName} refers to an unknown column`);
|
||||
}
|
||||
return {
|
||||
position: pair.position,
|
||||
sourceColumnId: source.id,
|
||||
sourceColumnName: source.name,
|
||||
targetColumnId: target.id,
|
||||
targetColumnName: target.name,
|
||||
};
|
||||
}).sort((a, b) => a.position - b.position);
|
||||
const key = `${observed.sourceTableName}\u0000${observed.constraintName}`;
|
||||
const current = existingByKey.get(key);
|
||||
const comparable = current && JSON.stringify({
|
||||
target: current.targetTableName,
|
||||
update: current.updateRule,
|
||||
delete: current.deleteRule,
|
||||
deferrable: current.deferrable,
|
||||
deferred: current.initiallyDeferred,
|
||||
columns: current.columns.map((pair) => [pair.position, pair.sourceColumnName, pair.targetColumnName]),
|
||||
});
|
||||
const nextComparable = JSON.stringify({
|
||||
target: observed.targetTableName,
|
||||
update: observed.updateRule,
|
||||
delete: observed.deleteRule,
|
||||
deferrable: observed.deferrable,
|
||||
deferred: observed.initiallyDeferred,
|
||||
columns: pairs.map((pair) => [pair.position, pair.sourceColumnName, pair.targetColumnName]),
|
||||
});
|
||||
const id = current?.id ?? randomUUID();
|
||||
this.relationships.set(id, {
|
||||
id,
|
||||
databaseId,
|
||||
constraintName: observed.constraintName,
|
||||
sourceTableId: sourceTable.id,
|
||||
sourceTableName: sourceTable.name,
|
||||
targetTableId: targetTable.id,
|
||||
targetTableName: targetTable.name,
|
||||
updateRule: observed.updateRule,
|
||||
deleteRule: observed.deleteRule,
|
||||
deferrable: observed.deferrable,
|
||||
initiallyDeferred: observed.initiallyDeferred,
|
||||
columns: pairs,
|
||||
lastSyncedDatabaseVersion: expectedDatabaseVersion,
|
||||
lastSyncedAt: now,
|
||||
createdAt: current?.createdAt ?? now,
|
||||
updatedAt: comparable === nextComparable ? (current?.updatedAt ?? now) : now,
|
||||
});
|
||||
if (!current) created += 1;
|
||||
else if (comparable !== nextComparable) updated += 1;
|
||||
}
|
||||
this.refreshForeignKeyFlags(databaseId);
|
||||
}
|
||||
|
||||
if (scope === "all") {
|
||||
this.records.set(databaseId, { ...database, schemaSyncedVersion: expectedDatabaseVersion, schemaSyncedAt: now });
|
||||
}
|
||||
return {
|
||||
tables: (await this.listTables(databaseId)).length,
|
||||
columns: [...this.columns.values()].filter((column) => this.tables.get(column.tableId)?.databaseId === databaseId).length,
|
||||
relationships: (await this.listRelationships(databaseId)).length,
|
||||
created,
|
||||
updated,
|
||||
deleted,
|
||||
};
|
||||
} catch (error) {
|
||||
this.tables.clear();
|
||||
this.columns.clear();
|
||||
this.relationships.clear();
|
||||
for (const [id, table] of tableBackup) this.tables.set(id, table);
|
||||
for (const [id, column] of columnBackup) this.columns.set(id, column);
|
||||
for (const [id, relationship] of relationshipBackup) this.relationships.set(id, relationship);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async createSyncRun(
|
||||
databaseId: string,
|
||||
scope: CatalogSyncScope,
|
||||
tableIds: readonly string[],
|
||||
requestedDatabaseVersion: number,
|
||||
): Promise<CatalogSyncRun> {
|
||||
const activeStates = new Set<CatalogSyncRun["state"]>(["queued", "running", "awaiting_confirmation", "applying"]);
|
||||
if ([...this.syncRuns.values()].some((run) => run.databaseId === databaseId && activeStates.has(run.state))) {
|
||||
throw new CatalogConflictError("A schema synchronization is already active for this database");
|
||||
}
|
||||
const now = new Date().toISOString();
|
||||
const run: CatalogSyncRun = {
|
||||
id: randomUUID(), databaseId, scope, tableIds: [...tableIds], state: "queued", phase: "queued",
|
||||
requestedDatabaseVersion, observedSnapshot: null, plannedDiff: null, confirmationToken: null,
|
||||
counts: {}, errorCode: null, errorMessage: null, cancelRequested: false,
|
||||
createdAt: now, startedAt: null, updatedAt: now, finishedAt: null, heartbeatAt: null,
|
||||
leaseOwner: null, leaseExpiresAt: null,
|
||||
};
|
||||
this.syncRuns.set(run.id, run);
|
||||
return structuredClone(run);
|
||||
}
|
||||
|
||||
async getSyncRun(runId: string): Promise<CatalogSyncRun | undefined> {
|
||||
const run = this.syncRuns.get(runId);
|
||||
return run ? structuredClone(run) : undefined;
|
||||
}
|
||||
|
||||
async claimSyncRun(runId: string, workerId: string, leaseExpiresAt: string): Promise<CatalogSyncRun | undefined> {
|
||||
const run = this.syncRuns.get(runId);
|
||||
if (!run || run.state !== "queued") return undefined;
|
||||
if (run.leaseOwner && run.leaseOwner !== workerId && run.leaseExpiresAt && run.leaseExpiresAt > new Date().toISOString()) {
|
||||
return undefined;
|
||||
}
|
||||
return await this.updateSyncRun(runId, {
|
||||
state: "running",
|
||||
startedAt: run.startedAt ?? new Date().toISOString(),
|
||||
heartbeatAt: new Date().toISOString(),
|
||||
leaseOwner: workerId,
|
||||
leaseExpiresAt,
|
||||
});
|
||||
}
|
||||
|
||||
async listSyncRuns(databaseId: string, limit = 20): Promise<CatalogSyncRun[]> {
|
||||
return [...this.syncRuns.values()].filter((run) => run.databaseId === databaseId)
|
||||
.sort((a, b) => b.createdAt.localeCompare(a.createdAt)).slice(0, limit).map((run) => structuredClone(run));
|
||||
}
|
||||
|
||||
async updateSyncRun(runId: string, update: CatalogSyncRunUpdate): Promise<CatalogSyncRun | undefined> {
|
||||
const current = this.syncRuns.get(runId);
|
||||
if (!current) return undefined;
|
||||
const updated = { ...current, ...structuredClone(update), updatedAt: new Date().toISOString() };
|
||||
this.syncRuns.set(runId, updated);
|
||||
return structuredClone(updated);
|
||||
}
|
||||
|
||||
async requestSyncRunCancellation(runId: string): Promise<CatalogSyncRun | undefined> {
|
||||
const run = this.syncRuns.get(runId);
|
||||
if (!run) return undefined;
|
||||
if (!["queued", "running", "awaiting_confirmation"].includes(run.state)) return structuredClone(run);
|
||||
return await this.updateSyncRun(runId, { cancelRequested: true });
|
||||
}
|
||||
|
||||
async appendSyncEvent(
|
||||
runId: string,
|
||||
level: CatalogSyncEvent["level"],
|
||||
eventType: string,
|
||||
message: string,
|
||||
data: Record<string, unknown> = {},
|
||||
): Promise<CatalogSyncEvent> {
|
||||
const events = this.syncEvents.get(runId) ?? [];
|
||||
const event: CatalogSyncEvent = {
|
||||
id: [...this.syncEvents.values()].reduce((count, values) => count + values.length, 0) + 1,
|
||||
runId, sequence: events.length + 1, level, eventType, message, data: structuredClone(data),
|
||||
createdAt: new Date().toISOString(),
|
||||
};
|
||||
events.push(event);
|
||||
this.syncEvents.set(runId, events);
|
||||
return structuredClone(event);
|
||||
}
|
||||
|
||||
async listSyncEvents(runId: string, afterSequence = 0): Promise<CatalogSyncEvent[]> {
|
||||
return (this.syncEvents.get(runId) ?? []).filter((event) => event.sequence > afterSequence).map((event) => structuredClone(event));
|
||||
}
|
||||
|
||||
async pruneSyncEvents(before: string): Promise<void> {
|
||||
for (const [runId, events] of this.syncEvents) {
|
||||
this.syncEvents.set(runId, events.filter((event) => event.createdAt >= before));
|
||||
}
|
||||
}
|
||||
|
||||
async interruptActiveSyncRuns(): Promise<void> {
|
||||
for (const run of this.syncRuns.values()) {
|
||||
if (["queued", "running", "awaiting_confirmation", "applying"].includes(run.state)) {
|
||||
await this.updateSyncRun(run.id, {
|
||||
state: "interrupted", phase: "completed", finishedAt: new Date().toISOString(),
|
||||
errorCode: "SYNC_INTERRUPTED", errorMessage: "Synchronization was interrupted by a service restart",
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async reconcileTables(
|
||||
databaseId: string,
|
||||
expectedDatabaseVersion: number,
|
||||
observed: readonly ObservedCatalogTable[],
|
||||
confirmedDeletedNames: readonly string[],
|
||||
): Promise<TableSyncRepositoryResult | undefined> {
|
||||
const database = this.records.get(databaseId);
|
||||
if (!database || database.version !== expectedDatabaseVersion) return undefined;
|
||||
const existing = await this.listTables(databaseId);
|
||||
const observedNames = new Set(observed.map((table) => table.name));
|
||||
const deletedNames = existing.filter((table) => !observedNames.has(table.name)).map((table) => table.name).sort();
|
||||
const confirmation = [...new Set(confirmedDeletedNames)].sort();
|
||||
if (deletedNames.length > 0 && JSON.stringify(deletedNames) !== JSON.stringify(confirmation)) {
|
||||
return { kind: "confirmation_required", deletedNames };
|
||||
}
|
||||
|
||||
const byName = new Map(existing.map((table) => [table.name, table]));
|
||||
let createdCount = 0;
|
||||
let updatedCount = 0;
|
||||
for (const observedTable of observed) {
|
||||
const current = byName.get(observedTable.name);
|
||||
const now = new Date().toISOString();
|
||||
if (!current) {
|
||||
const created: CatalogTable = {
|
||||
id: randomUUID(),
|
||||
databaseId,
|
||||
name: observedTable.name,
|
||||
sourceComment: observedTable.sourceComment,
|
||||
description: null,
|
||||
generatedDescription: null,
|
||||
lastSyncedDatabaseVersion: expectedDatabaseVersion,
|
||||
lastSyncedAt: now,
|
||||
version: 1,
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
};
|
||||
this.tables.set(created.id, created);
|
||||
createdCount += 1;
|
||||
} else if (current.sourceComment !== observedTable.sourceComment) {
|
||||
this.tables.set(current.id, {
|
||||
...current,
|
||||
sourceComment: observedTable.sourceComment,
|
||||
version: current.version + 1,
|
||||
updatedAt: now,
|
||||
});
|
||||
updatedCount += 1;
|
||||
}
|
||||
}
|
||||
for (const table of existing) {
|
||||
if (deletedNames.includes(table.name)) this.deleteTable(table.id);
|
||||
}
|
||||
return {
|
||||
kind: "applied",
|
||||
createdCount,
|
||||
updatedCount,
|
||||
deletedCount: deletedNames.length,
|
||||
tables: await this.listTables(databaseId),
|
||||
};
|
||||
}
|
||||
|
||||
private assertSnapshotCapability(scope: CatalogSyncScope, snapshot: ObservedSchemaSnapshot): void {
|
||||
const required = scope === "all" ? ["tables", "columns", "relationships"] as const : [scope] as const;
|
||||
for (const capability of required) {
|
||||
if (snapshot.capabilities[capability] !== "available") {
|
||||
throw new CatalogConnectorError(`Schema introspection capability '${capability}' is unavailable`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private deleteTable(tableId: string): void {
|
||||
this.tables.delete(tableId);
|
||||
for (const column of [...this.columns.values()]) if (column.tableId === tableId) this.deleteColumn(column.id);
|
||||
for (const relationship of [...this.relationships.values()]) {
|
||||
if (relationship.sourceTableId === tableId || relationship.targetTableId === tableId) {
|
||||
this.relationships.delete(relationship.id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private deleteColumn(columnId: string): void {
|
||||
this.columns.delete(columnId);
|
||||
for (const relationship of [...this.relationships.values()]) {
|
||||
if (relationship.columns.some((pair) => pair.sourceColumnId === columnId || pair.targetColumnId === columnId)) {
|
||||
this.relationships.delete(relationship.id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private refreshForeignKeyFlags(databaseId: string): void {
|
||||
const counts = new Map<string, number>();
|
||||
for (const relationship of this.relationships.values()) {
|
||||
if (relationship.databaseId !== databaseId) continue;
|
||||
for (const pair of relationship.columns) counts.set(pair.sourceColumnId, (counts.get(pair.sourceColumnId) ?? 0) + 1);
|
||||
}
|
||||
for (const column of this.columns.values()) {
|
||||
if (this.tables.get(column.tableId)?.databaseId !== databaseId) continue;
|
||||
const foreignKeyCount = counts.get(column.id) ?? 0;
|
||||
this.columns.set(column.id, { ...column, isForeignKey: foreignKeyCount > 0, foreignKeyCount });
|
||||
}
|
||||
}
|
||||
|
||||
async available(): Promise<boolean> { return true; }
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
import { CamelCasePlugin, Kysely, PostgresDialect } from "kysely";
|
||||
import { Migrator, type MigrationProvider } from "kysely/migration";
|
||||
import { Pool } from "pg";
|
||||
import { readFile } from "node:fs/promises";
|
||||
import type { CatalogDatabase } from "./repository.js";
|
||||
import * as initialMigration from "./migrations/001_workspace_databases.js";
|
||||
import * as catalogTablesMigration from "./migrations/002_catalog_tables.js";
|
||||
import * as catalogSchemaSyncMigration from "./migrations/003_catalog_schema_sync.js";
|
||||
import * as catalogRuntimeSequencePrivilegesMigration from "./migrations/004_catalog_runtime_sequence_privileges.js";
|
||||
|
||||
const connectionString = process.env.THT_CATALOG_MIGRATOR_DATABASE_URL;
|
||||
const host = process.env.THT_CATALOG_DB_HOST;
|
||||
const database = process.env.THT_CATALOG_DB_NAME;
|
||||
const user = process.env.THT_CATALOG_MIGRATOR_USER;
|
||||
const passwordFile = process.env.THT_CATALOG_MIGRATOR_PASSWORD_FILE;
|
||||
if (!connectionString && (!host || !database || !user || !passwordFile)) {
|
||||
throw new Error("catalog migrator database configuration is required");
|
||||
}
|
||||
const pool = new Pool(connectionString ? { connectionString, max: 1 } : {
|
||||
host,
|
||||
port: Number(process.env.THT_CATALOG_DB_PORT ?? 5432),
|
||||
database,
|
||||
user,
|
||||
password: async () => (await readFile(passwordFile!, "utf8")).trim(),
|
||||
max: 1,
|
||||
});
|
||||
|
||||
const db = new Kysely<CatalogDatabase>({
|
||||
dialect: new PostgresDialect({ pool }),
|
||||
plugins: [new CamelCasePlugin()],
|
||||
});
|
||||
const provider: MigrationProvider = {
|
||||
async getMigrations() {
|
||||
return {
|
||||
"001_workspace_databases": initialMigration,
|
||||
"002_catalog_tables": catalogTablesMigration,
|
||||
"003_catalog_schema_sync": catalogSchemaSyncMigration,
|
||||
"004_catalog_runtime_sequence_privileges": catalogRuntimeSequencePrivilegesMigration,
|
||||
};
|
||||
},
|
||||
};
|
||||
|
||||
try {
|
||||
const result = await new Migrator({ db, provider }).migrateToLatest();
|
||||
for (const item of result.results ?? []) {
|
||||
process.stdout.write(`${item.migrationName}: ${item.status}\n`);
|
||||
}
|
||||
if (result.error) throw result.error;
|
||||
} finally {
|
||||
await db.destroy();
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
import { sql, type Kysely } from "kysely";
|
||||
import type { CatalogDatabase } from "../repository.js";
|
||||
|
||||
export async function up(db: Kysely<CatalogDatabase>): Promise<void> {
|
||||
await db.schema.createTable("workspace_databases")
|
||||
.addColumn("id", "uuid", (column) => column.primaryKey())
|
||||
.addColumn("workspace_id", "text", (column) => column.notNull().unique())
|
||||
.addColumn("engine", "text", (column) => column.notNull())
|
||||
.addColumn("database_name", "text", (column) => column.notNull())
|
||||
.addColumn("schema_name", "text", (column) => column.notNull())
|
||||
.addColumn("version", "integer", (column) => column.notNull().defaultTo(1))
|
||||
.addColumn("created_at", "timestamptz", (column) => column.notNull().defaultTo(sql`now()`))
|
||||
.addColumn("updated_at", "timestamptz", (column) => column.notNull().defaultTo(sql`now()`))
|
||||
.addCheckConstraint("workspace_databases_engine_check", sql`engine = 'postgres'`)
|
||||
.addCheckConstraint("workspace_databases_version_check", sql`version > 0`)
|
||||
.execute();
|
||||
|
||||
await db.schema.createTable("database_bindings")
|
||||
.addColumn("database_id", "uuid", (column) => column.primaryKey()
|
||||
.references("workspace_databases.id").onDelete("cascade"))
|
||||
.addColumn("transport", "text", (column) => column.notNull())
|
||||
.addColumn("host", "text")
|
||||
.addColumn("port", "integer")
|
||||
.addColumn("username", "text")
|
||||
.addColumn("base_url", "text")
|
||||
.addColumn("rest_path", "text")
|
||||
.addColumn("rest_auth", "text")
|
||||
.addColumn("tls_servername", "text")
|
||||
.addColumn("ssh_host", "text")
|
||||
.addColumn("ssh_port", "integer")
|
||||
.addColumn("ssh_username", "text")
|
||||
.addColumn("ssh_target_host", "text")
|
||||
.addColumn("ssh_target_port", "integer")
|
||||
.addColumn("connection_status", "text", (column) => column.notNull().defaultTo("untested"))
|
||||
.addColumn("tested_version", "integer")
|
||||
.addColumn("last_tested_at", "timestamptz")
|
||||
.addColumn("last_error_code", "text")
|
||||
.addColumn("last_error_message", "text")
|
||||
.addColumn("updated_at", "timestamptz", (column) => column.notNull().defaultTo(sql`now()`))
|
||||
.addCheckConstraint("database_bindings_transport_check", sql`transport in ('postgres_direct', 'rest_api', 'ssh_tunnel')`)
|
||||
.addCheckConstraint("database_bindings_status_check", sql`connection_status in ('untested', 'reachable', 'failed')`)
|
||||
.addCheckConstraint("database_bindings_port_check", sql`port is null or port between 1 and 65535`)
|
||||
.addCheckConstraint("database_bindings_ssh_port_check", sql`ssh_port is null or ssh_port between 1 and 65535`)
|
||||
.addCheckConstraint("database_bindings_ssh_target_port_check", sql`ssh_target_port is null or ssh_target_port between 1 and 65535`)
|
||||
.addCheckConstraint("database_bindings_transport_fields_check", sql`
|
||||
(transport = 'postgres_direct' and host is not null and port is not null and username is not null)
|
||||
or (transport = 'rest_api' and base_url is not null and rest_path is not null and rest_auth is not null)
|
||||
or (transport = 'ssh_tunnel' and username is not null and ssh_host is not null
|
||||
and ssh_port is not null and ssh_username is not null and ssh_target_host is not null
|
||||
and ssh_target_port is not null)
|
||||
`)
|
||||
.execute();
|
||||
}
|
||||
|
||||
export async function down(db: Kysely<CatalogDatabase>): Promise<void> {
|
||||
await db.schema.dropTable("database_bindings").execute();
|
||||
await db.schema.dropTable("workspace_databases").execute();
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
import { sql, type Kysely } from "kysely";
|
||||
import type { CatalogDatabase } from "../repository.js";
|
||||
|
||||
export async function up(db: Kysely<CatalogDatabase>): Promise<void> {
|
||||
await db.schema.createTable("catalog_tables")
|
||||
.addColumn("id", "uuid", (column) => column.primaryKey())
|
||||
.addColumn("database_id", "uuid", (column) => column.notNull()
|
||||
.references("workspace_databases.id").onDelete("cascade"))
|
||||
.addColumn("name", "text", (column) => column.notNull())
|
||||
.addColumn("source_comment", "text")
|
||||
.addColumn("description", "text")
|
||||
.addColumn("generated_description", "text")
|
||||
.addColumn("version", "integer", (column) => column.notNull().defaultTo(1))
|
||||
.addColumn("created_at", "timestamptz", (column) => column.notNull().defaultTo(sql`now()`))
|
||||
.addColumn("updated_at", "timestamptz", (column) => column.notNull().defaultTo(sql`now()`))
|
||||
.addUniqueConstraint("catalog_tables_database_name_key", ["database_id", "name"])
|
||||
.addCheckConstraint("catalog_tables_name_check", sql`char_length(name) between 1 and 128`)
|
||||
.addCheckConstraint("catalog_tables_version_check", sql`version > 0`)
|
||||
.execute();
|
||||
|
||||
await db.schema.createIndex("catalog_tables_database_id_idx")
|
||||
.on("catalog_tables").column("database_id").execute();
|
||||
}
|
||||
|
||||
export async function down(db: Kysely<CatalogDatabase>): Promise<void> {
|
||||
await db.schema.dropTable("catalog_tables").execute();
|
||||
}
|
||||
@@ -0,0 +1,136 @@
|
||||
import { sql, type Kysely } from "kysely";
|
||||
import type { CatalogDatabase } from "../repository.js";
|
||||
|
||||
export async function up(db: Kysely<CatalogDatabase>): Promise<void> {
|
||||
await db.schema.alterTable("workspace_databases")
|
||||
.addColumn("schema_synced_version", "integer")
|
||||
.addColumn("schema_synced_at", "timestamptz")
|
||||
.execute();
|
||||
|
||||
await db.schema.alterTable("catalog_tables")
|
||||
.addColumn("last_synced_database_version", "integer")
|
||||
.addColumn("last_synced_at", "timestamptz")
|
||||
.execute();
|
||||
|
||||
await db.schema.createTable("catalog_columns")
|
||||
.addColumn("id", "uuid", (column) => column.primaryKey())
|
||||
.addColumn("table_id", "uuid", (column) => column.notNull()
|
||||
.references("catalog_tables.id").onDelete("cascade"))
|
||||
.addColumn("name", "text", (column) => column.notNull())
|
||||
.addColumn("ordinal_position", "integer", (column) => column.notNull())
|
||||
.addColumn("data_type", "text", (column) => column.notNull())
|
||||
.addColumn("is_nullable", "boolean", (column) => column.notNull())
|
||||
.addColumn("default_expression", "text")
|
||||
.addColumn("primary_key_position", "integer")
|
||||
.addColumn("source_comment", "text")
|
||||
.addColumn("description", "text")
|
||||
.addColumn("generated_description", "text")
|
||||
.addColumn("last_synced_database_version", "integer")
|
||||
.addColumn("last_synced_at", "timestamptz")
|
||||
.addColumn("version", "integer", (column) => column.notNull().defaultTo(1))
|
||||
.addColumn("created_at", "timestamptz", (column) => column.notNull().defaultTo(sql`now()`))
|
||||
.addColumn("updated_at", "timestamptz", (column) => column.notNull().defaultTo(sql`now()`))
|
||||
.addUniqueConstraint("catalog_columns_table_name_key", ["table_id", "name"])
|
||||
.addCheckConstraint("catalog_columns_name_check", sql`char_length(name) between 1 and 128`)
|
||||
.addCheckConstraint("catalog_columns_ordinal_check", sql`ordinal_position > 0`)
|
||||
.addCheckConstraint("catalog_columns_pk_position_check", sql`primary_key_position is null or primary_key_position > 0`)
|
||||
.addCheckConstraint("catalog_columns_version_check", sql`version > 0`)
|
||||
.execute();
|
||||
await db.schema.createIndex("catalog_columns_table_id_idx")
|
||||
.on("catalog_columns").column("table_id").execute();
|
||||
|
||||
await db.schema.createTable("catalog_relationships")
|
||||
.addColumn("id", "uuid", (column) => column.primaryKey())
|
||||
.addColumn("database_id", "uuid", (column) => column.notNull()
|
||||
.references("workspace_databases.id").onDelete("cascade"))
|
||||
.addColumn("constraint_name", "text", (column) => column.notNull())
|
||||
.addColumn("source_table_id", "uuid", (column) => column.notNull()
|
||||
.references("catalog_tables.id").onDelete("cascade"))
|
||||
.addColumn("target_table_id", "uuid", (column) => column.notNull()
|
||||
.references("catalog_tables.id").onDelete("cascade"))
|
||||
.addColumn("update_rule", "text", (column) => column.notNull())
|
||||
.addColumn("delete_rule", "text", (column) => column.notNull())
|
||||
.addColumn("deferrable", "boolean", (column) => column.notNull().defaultTo(false))
|
||||
.addColumn("initially_deferred", "boolean", (column) => column.notNull().defaultTo(false))
|
||||
.addColumn("last_synced_database_version", "integer")
|
||||
.addColumn("last_synced_at", "timestamptz")
|
||||
.addColumn("created_at", "timestamptz", (column) => column.notNull().defaultTo(sql`now()`))
|
||||
.addColumn("updated_at", "timestamptz", (column) => column.notNull().defaultTo(sql`now()`))
|
||||
.addUniqueConstraint("catalog_relationships_source_constraint_key", ["source_table_id", "constraint_name"])
|
||||
.execute();
|
||||
await db.schema.createIndex("catalog_relationships_database_id_idx")
|
||||
.on("catalog_relationships").column("database_id").execute();
|
||||
|
||||
await db.schema.createTable("catalog_relationship_columns")
|
||||
.addColumn("relationship_id", "uuid", (column) => column.notNull()
|
||||
.references("catalog_relationships.id").onDelete("cascade"))
|
||||
.addColumn("position", "integer", (column) => column.notNull())
|
||||
.addColumn("source_column_id", "uuid", (column) => column.notNull()
|
||||
.references("catalog_columns.id").onDelete("cascade"))
|
||||
.addColumn("target_column_id", "uuid", (column) => column.notNull()
|
||||
.references("catalog_columns.id").onDelete("cascade"))
|
||||
.addPrimaryKeyConstraint("catalog_relationship_columns_pkey", ["relationship_id", "position"])
|
||||
.addCheckConstraint("catalog_relationship_columns_position_check", sql`position > 0`)
|
||||
.execute();
|
||||
|
||||
await db.schema.createTable("catalog_sync_runs")
|
||||
.addColumn("id", "uuid", (column) => column.primaryKey())
|
||||
.addColumn("database_id", "uuid", (column) => column.notNull()
|
||||
.references("workspace_databases.id").onDelete("cascade"))
|
||||
.addColumn("scope", "text", (column) => column.notNull())
|
||||
.addColumn("table_ids", "jsonb", (column) => column.notNull().defaultTo(sql`'[]'::jsonb`))
|
||||
.addColumn("state", "text", (column) => column.notNull())
|
||||
.addColumn("phase", "text", (column) => column.notNull())
|
||||
.addColumn("requested_database_version", "integer", (column) => column.notNull())
|
||||
.addColumn("observed_snapshot", "jsonb")
|
||||
.addColumn("planned_diff", "jsonb")
|
||||
.addColumn("confirmation_token", "text")
|
||||
.addColumn("counts", "jsonb", (column) => column.notNull().defaultTo(sql`'{}'::jsonb`))
|
||||
.addColumn("error_code", "text")
|
||||
.addColumn("error_message", "text")
|
||||
.addColumn("cancel_requested", "boolean", (column) => column.notNull().defaultTo(false))
|
||||
.addColumn("created_at", "timestamptz", (column) => column.notNull().defaultTo(sql`now()`))
|
||||
.addColumn("started_at", "timestamptz")
|
||||
.addColumn("updated_at", "timestamptz", (column) => column.notNull().defaultTo(sql`now()`))
|
||||
.addColumn("finished_at", "timestamptz")
|
||||
.addColumn("heartbeat_at", "timestamptz")
|
||||
.addColumn("lease_owner", "text")
|
||||
.addColumn("lease_expires_at", "timestamptz")
|
||||
.execute();
|
||||
await db.schema.createIndex("catalog_sync_runs_database_created_idx")
|
||||
.on("catalog_sync_runs").columns(["database_id", "created_at"]).execute();
|
||||
await sql`CREATE UNIQUE INDEX catalog_sync_runs_one_active_per_database
|
||||
ON catalog_sync_runs (database_id)
|
||||
WHERE state IN ('queued', 'running', 'awaiting_confirmation', 'applying')`.execute(db);
|
||||
|
||||
await db.schema.createTable("catalog_sync_events")
|
||||
.addColumn("id", "bigserial", (column) => column.primaryKey())
|
||||
.addColumn("run_id", "uuid", (column) => column.notNull()
|
||||
.references("catalog_sync_runs.id").onDelete("cascade"))
|
||||
.addColumn("sequence", "integer", (column) => column.notNull())
|
||||
.addColumn("level", "text", (column) => column.notNull())
|
||||
.addColumn("event_type", "text", (column) => column.notNull())
|
||||
.addColumn("message", "text", (column) => column.notNull())
|
||||
.addColumn("data", "jsonb", (column) => column.notNull().defaultTo(sql`'{}'::jsonb`))
|
||||
.addColumn("created_at", "timestamptz", (column) => column.notNull().defaultTo(sql`now()`))
|
||||
.addUniqueConstraint("catalog_sync_events_run_sequence_key", ["run_id", "sequence"])
|
||||
.execute();
|
||||
await db.schema.createIndex("catalog_sync_events_run_id_idx")
|
||||
.on("catalog_sync_events").columns(["run_id", "sequence"]).execute();
|
||||
}
|
||||
|
||||
export async function down(db: Kysely<CatalogDatabase>): Promise<void> {
|
||||
await db.schema.dropTable("catalog_sync_events").execute();
|
||||
await db.schema.dropTable("catalog_sync_runs").execute();
|
||||
await db.schema.dropTable("catalog_relationship_columns").execute();
|
||||
await db.schema.dropTable("catalog_relationships").execute();
|
||||
await db.schema.dropTable("catalog_columns").execute();
|
||||
await db.schema.alterTable("catalog_tables")
|
||||
.dropColumn("last_synced_database_version")
|
||||
.dropColumn("last_synced_at")
|
||||
.execute();
|
||||
await db.schema.alterTable("workspace_databases")
|
||||
.dropColumn("schema_synced_version")
|
||||
.dropColumn("schema_synced_at")
|
||||
.execute();
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
import { type Kysely, sql } from "kysely";
|
||||
import type { CatalogDatabase } from "../repository.js";
|
||||
|
||||
/**
|
||||
* `catalog_sync_events.id` is the first catalog-owned identity sequence.
|
||||
* Table default privileges do not cover sequences, and without USAGE the
|
||||
* runtime can create a run but cannot append its first event.
|
||||
*/
|
||||
export async function up(db: Kysely<CatalogDatabase>): Promise<void> {
|
||||
await sql`DO $catalog_privileges$
|
||||
BEGIN
|
||||
IF EXISTS (SELECT 1 FROM pg_catalog.pg_roles WHERE rolname = 'thothii_catalog_runtime') THEN
|
||||
EXECUTE 'GRANT USAGE, SELECT ON SEQUENCE catalog_sync_events_id_seq TO thothii_catalog_runtime';
|
||||
EXECUTE 'ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT USAGE, SELECT ON SEQUENCES TO thothii_catalog_runtime';
|
||||
END IF;
|
||||
END
|
||||
$catalog_privileges$`.execute(db);
|
||||
}
|
||||
|
||||
export async function down(db: Kysely<CatalogDatabase>): Promise<void> {
|
||||
await sql`DO $catalog_privileges$
|
||||
BEGIN
|
||||
IF EXISTS (SELECT 1 FROM pg_catalog.pg_roles WHERE rolname = 'thothii_catalog_runtime') THEN
|
||||
EXECUTE 'ALTER DEFAULT PRIVILEGES IN SCHEMA public REVOKE USAGE, SELECT ON SEQUENCES FROM thothii_catalog_runtime';
|
||||
EXECUTE 'REVOKE USAGE, SELECT ON SEQUENCE catalog_sync_events_id_seq FROM thothii_catalog_runtime';
|
||||
END IF;
|
||||
END
|
||||
$catalog_privileges$`.execute(db);
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
import { CatalogOperationInProgressError } from "./types.js";
|
||||
|
||||
/** Serializes connection tests and schema synchronization for each catalog database. */
|
||||
export class CatalogOperationCoordinator {
|
||||
private readonly active = new Set<string>();
|
||||
|
||||
reserve(databaseId: string): () => void {
|
||||
if (this.active.has(databaseId)) {
|
||||
throw new CatalogOperationInProgressError("A database operation is already in progress");
|
||||
}
|
||||
this.active.add(databaseId);
|
||||
let released = false;
|
||||
return () => {
|
||||
if (released) return;
|
||||
released = true;
|
||||
this.active.delete(databaseId);
|
||||
};
|
||||
}
|
||||
|
||||
async run<T>(databaseId: string, operation: () => Promise<T>): Promise<T> {
|
||||
const release = this.reserve(databaseId);
|
||||
try {
|
||||
return await operation();
|
||||
} finally {
|
||||
release();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,258 @@
|
||||
import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process";
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { Duplex } from "node:stream";
|
||||
import { setTimeout as delay } from "node:timers/promises";
|
||||
import { Client, type ClientConfig } from "pg";
|
||||
import type { WorkspaceSecretStore } from "../workspaces/secret-store.js";
|
||||
import { CATALOG_SECRET_IDS } from "./secrets.js";
|
||||
import { CatalogConnectorError, type WorkspaceDatabase } from "./types.js";
|
||||
|
||||
export interface CatalogDatabaseClient {
|
||||
query(sql: string, values: readonly unknown[]): Promise<{ rows: Array<Record<string, unknown>> }>;
|
||||
end(): Promise<void>;
|
||||
}
|
||||
|
||||
export interface CatalogPostgresAccess {
|
||||
connect(database: WorkspaceDatabase, signal: AbortSignal): Promise<CatalogDatabaseClient>;
|
||||
}
|
||||
|
||||
type SpawnSsh = (
|
||||
command: string,
|
||||
args: readonly string[],
|
||||
options: { env: NodeJS.ProcessEnv },
|
||||
) => ChildProcessWithoutNullStreams;
|
||||
|
||||
interface AccessDependencies {
|
||||
createClient?: (config: ClientConfig) => Client;
|
||||
spawnSsh?: SpawnSsh;
|
||||
sshBinary?: string;
|
||||
askpassPath?: string;
|
||||
connectTimeoutMs?: number;
|
||||
}
|
||||
|
||||
function required(value: string | number | undefined): string | number {
|
||||
if (value === undefined || value === "") throw new CatalogConnectorError("Database binding is incomplete");
|
||||
return value;
|
||||
}
|
||||
|
||||
function connectionSsl(ca: string | undefined, servername: string | undefined): ClientConfig["ssl"] {
|
||||
if (!ca && !servername) return false;
|
||||
return {
|
||||
...(ca ? { ca } : {}),
|
||||
...(servername ? { servername } : {}),
|
||||
rejectUnauthorized: true,
|
||||
};
|
||||
}
|
||||
|
||||
export function buildSshArguments(input: {
|
||||
sshHost: string;
|
||||
sshPort: number;
|
||||
sshUsername: string;
|
||||
targetHost: string;
|
||||
targetPort: number;
|
||||
privateKeyFile: string;
|
||||
knownHostsFile: string;
|
||||
passphraseFile?: string;
|
||||
connectTimeoutMs: number;
|
||||
}): string[] {
|
||||
const batchMode = input.passphraseFile ? "no" : "yes";
|
||||
return [
|
||||
"-F", "/dev/null",
|
||||
"-T",
|
||||
"-o", `BatchMode=${batchMode}`,
|
||||
"-o", "StrictHostKeyChecking=yes",
|
||||
"-o", `UserKnownHostsFile=${input.knownHostsFile}`,
|
||||
"-o", "GlobalKnownHostsFile=/dev/null",
|
||||
"-o", "IdentitiesOnly=yes",
|
||||
"-o", "IdentityAgent=none",
|
||||
"-o", `IdentityFile=${input.privateKeyFile}`,
|
||||
"-o", "PreferredAuthentications=publickey",
|
||||
"-o", "PasswordAuthentication=no",
|
||||
"-o", "KbdInteractiveAuthentication=no",
|
||||
"-o", "ConnectionAttempts=1",
|
||||
"-o", `ConnectTimeout=${Math.max(1, Math.ceil(input.connectTimeoutMs / 1_000))}`,
|
||||
"-o", "ServerAliveInterval=5",
|
||||
"-o", "ServerAliveCountMax=1",
|
||||
"-o", "NumberOfPasswordPrompts=1",
|
||||
"-o", "RequestTTY=no",
|
||||
"-o", "LogLevel=ERROR",
|
||||
"-p", String(input.sshPort),
|
||||
"-W", `${input.targetHost}:${input.targetPort}`,
|
||||
"--", `${input.sshUsername}@${input.sshHost}`,
|
||||
];
|
||||
}
|
||||
|
||||
async function stopChild(child: ChildProcessWithoutNullStreams): Promise<void> {
|
||||
if (child.exitCode !== null || child.signalCode !== null) return;
|
||||
child.kill("SIGTERM");
|
||||
await Promise.race([
|
||||
new Promise<void>((resolve) => child.once("exit", () => resolve())),
|
||||
delay(500).then(() => undefined),
|
||||
]);
|
||||
if (child.exitCode === null && child.signalCode === null) child.kill("SIGKILL");
|
||||
}
|
||||
|
||||
function sshDuplex(child: ChildProcessWithoutNullStreams): Duplex {
|
||||
let ended = false;
|
||||
let stream: Duplex;
|
||||
const forward = () => {
|
||||
let chunk: Buffer | string | null;
|
||||
while ((chunk = child.stdout.read() as Buffer | string | null) !== null) {
|
||||
if (!stream.push(chunk)) break;
|
||||
}
|
||||
};
|
||||
const finish = () => {
|
||||
if (ended) return;
|
||||
ended = true;
|
||||
stream.push(null);
|
||||
};
|
||||
const fail = (error: Error) => stream.destroy(error);
|
||||
stream = new Duplex({
|
||||
read: forward,
|
||||
write: (chunk, encoding, callback) => child.stdin.write(chunk, encoding, callback),
|
||||
final: (callback) => child.stdin.end(callback),
|
||||
destroy: (error, callback) => {
|
||||
child.stdout.off("readable", forward);
|
||||
child.stdout.off("end", finish);
|
||||
child.stdout.off("error", fail);
|
||||
child.stdin.off("error", fail);
|
||||
callback(error);
|
||||
},
|
||||
});
|
||||
child.stdout.on("readable", forward);
|
||||
child.stdout.once("end", finish);
|
||||
child.stdout.once("error", fail);
|
||||
child.stdin.once("error", fail);
|
||||
return stream;
|
||||
}
|
||||
|
||||
/**
|
||||
* Deep connection module for direct and SSH-forwarded PostgreSQL access. It owns secret leases,
|
||||
* TLS, OpenSSH lifecycle, abort propagation, and pg cleanup behind one connect interface.
|
||||
*/
|
||||
export class ConcreteCatalogPostgresAccess implements CatalogPostgresAccess {
|
||||
private readonly createClient: (config: ClientConfig) => Client;
|
||||
private readonly spawnSsh: SpawnSsh;
|
||||
private readonly sshBinary: string;
|
||||
private readonly askpassPath: string;
|
||||
private readonly connectTimeoutMs: number;
|
||||
|
||||
constructor(
|
||||
private readonly secretStore: WorkspaceSecretStore,
|
||||
dependencies: AccessDependencies = {},
|
||||
) {
|
||||
this.createClient = dependencies.createClient ?? ((config) => new Client(config));
|
||||
this.spawnSsh = dependencies.spawnSsh ?? ((command, args, options) => (
|
||||
spawn(command, [...args], { ...options, stdio: ["pipe", "pipe", "pipe"] })
|
||||
));
|
||||
this.sshBinary = dependencies.sshBinary ?? process.env.THT_SSH_BIN ?? "ssh";
|
||||
this.askpassPath = dependencies.askpassPath
|
||||
?? process.env.THT_SSH_ASKPASS_BIN
|
||||
?? fileURLToPath(new URL("../../scripts/ssh-askpass.mjs", import.meta.url));
|
||||
this.connectTimeoutMs = dependencies.connectTimeoutMs ?? 5_000;
|
||||
}
|
||||
|
||||
async connect(database: WorkspaceDatabase, signal: AbortSignal): Promise<CatalogDatabaseClient> {
|
||||
if (database.binding.transport === "rest_api") {
|
||||
throw new CatalogConnectorError("REST is not a PostgreSQL wire binding");
|
||||
}
|
||||
const ids: string[] = [CATALOG_SECRET_IDS.password, CATALOG_SECRET_IDS.tlsCa];
|
||||
if (database.binding.transport === "ssh_tunnel") {
|
||||
ids.push(
|
||||
CATALOG_SECRET_IDS.sshPrivateKey,
|
||||
CATALOG_SECRET_IDS.sshPrivateKeyPassphrase,
|
||||
CATALOG_SECRET_IDS.sshKnownHosts,
|
||||
);
|
||||
}
|
||||
const materialized = this.secretStore.materialize(database.workspaceId, ids);
|
||||
let child: ChildProcessWithoutNullStreams | undefined;
|
||||
let stream: Duplex | undefined;
|
||||
let client: Client | undefined;
|
||||
let ended = false;
|
||||
const close = async () => {
|
||||
if (ended) return;
|
||||
ended = true;
|
||||
signal.removeEventListener("abort", abort);
|
||||
if (client) await client.end().catch(() => undefined);
|
||||
stream?.destroy();
|
||||
if (child) await stopChild(child);
|
||||
materialized.release();
|
||||
};
|
||||
const abort = () => { void close(); };
|
||||
|
||||
try {
|
||||
const passwordFile = materialized.files.get(CATALOG_SECRET_IDS.password);
|
||||
if (!passwordFile) throw new CatalogConnectorError("Database password is not configured");
|
||||
const password = await readFile(passwordFile, "utf8");
|
||||
const tlsCaFile = materialized.files.get(CATALOG_SECRET_IDS.tlsCa);
|
||||
const tlsCa = tlsCaFile ? await readFile(tlsCaFile, "utf8") : undefined;
|
||||
let host: string;
|
||||
let port: number;
|
||||
|
||||
if (database.binding.transport === "ssh_tunnel") {
|
||||
const privateKeyFile = materialized.files.get(CATALOG_SECRET_IDS.sshPrivateKey);
|
||||
const knownHostsFile = materialized.files.get(CATALOG_SECRET_IDS.sshKnownHosts);
|
||||
if (!privateKeyFile || !knownHostsFile) {
|
||||
throw new CatalogConnectorError("SSH private key and known hosts are required");
|
||||
}
|
||||
const passphraseFile = materialized.files.get(CATALOG_SECRET_IDS.sshPrivateKeyPassphrase);
|
||||
host = String(required(database.binding.sshTargetHost));
|
||||
port = Number(required(database.binding.sshTargetPort));
|
||||
const args = buildSshArguments({
|
||||
sshHost: String(required(database.binding.sshHost)),
|
||||
sshPort: Number(required(database.binding.sshPort)),
|
||||
sshUsername: String(required(database.binding.sshUsername)),
|
||||
targetHost: host,
|
||||
targetPort: port,
|
||||
privateKeyFile,
|
||||
knownHostsFile,
|
||||
passphraseFile,
|
||||
connectTimeoutMs: this.connectTimeoutMs,
|
||||
});
|
||||
child = this.spawnSsh(this.sshBinary, args, {
|
||||
env: {
|
||||
...process.env,
|
||||
LC_ALL: "C",
|
||||
...(passphraseFile ? {
|
||||
DISPLAY: "thothii",
|
||||
SSH_ASKPASS: this.askpassPath,
|
||||
SSH_ASKPASS_REQUIRE: "force",
|
||||
THT_SSH_PASSPHRASE_FILE: passphraseFile,
|
||||
} : {}),
|
||||
},
|
||||
});
|
||||
stream = sshDuplex(child);
|
||||
child.once("error", () => stream?.destroy(new CatalogConnectorError("SSH process failed")));
|
||||
child.once("exit", (code) => {
|
||||
if (!ended && code !== 0) stream?.destroy(new CatalogConnectorError("SSH tunnel failed"));
|
||||
});
|
||||
child.stderr.on("data", () => undefined);
|
||||
} else {
|
||||
host = String(required(database.binding.host));
|
||||
port = Number(required(database.binding.port));
|
||||
}
|
||||
|
||||
client = this.createClient({
|
||||
host,
|
||||
port,
|
||||
database: database.databaseName,
|
||||
user: String(required(database.binding.username)),
|
||||
password,
|
||||
ssl: connectionSsl(tlsCa, database.binding.tlsServername),
|
||||
connectionTimeoutMillis: this.connectTimeoutMs,
|
||||
...(stream ? { stream: () => stream } : {}),
|
||||
});
|
||||
signal.addEventListener("abort", abort, { once: true });
|
||||
await client.connect();
|
||||
return {
|
||||
query: async (sql, values) => await client!.query(sql, [...values]),
|
||||
end: close,
|
||||
};
|
||||
} catch (error) {
|
||||
await close();
|
||||
if (error instanceof CatalogConnectorError) throw error;
|
||||
throw new CatalogConnectorError("PostgreSQL connector failed");
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,498 @@
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { z } from "zod";
|
||||
import type { WorkspaceSecretStore } from "../workspaces/secret-store.js";
|
||||
import type { CatalogPostgresAccess } from "./postgres-access.js";
|
||||
import { CATALOG_SECRET_IDS } from "./secrets.js";
|
||||
import {
|
||||
CatalogConnectorError,
|
||||
CatalogSchemaCapabilityUnavailableError,
|
||||
type CatalogSyncPhase,
|
||||
type ObservedCatalogColumn,
|
||||
type ObservedCatalogRelationship,
|
||||
type ObservedCatalogTable,
|
||||
type ObservedSchemaSnapshot,
|
||||
type WorkspaceDatabase,
|
||||
} from "./types.js";
|
||||
|
||||
export type CatalogSchemaScanProgress = (
|
||||
phase: Extract<CatalogSyncPhase, "connecting" | "scanning_tables" | "scanning_columns" | "scanning_relationships">,
|
||||
counts?: { tables?: number; columns?: number; relationships?: number },
|
||||
) => Promise<void> | void;
|
||||
|
||||
export interface CatalogSchemaIntrospector {
|
||||
scan(
|
||||
database: WorkspaceDatabase,
|
||||
signal: AbortSignal,
|
||||
progress?: CatalogSchemaScanProgress,
|
||||
): Promise<ObservedSchemaSnapshot>;
|
||||
}
|
||||
|
||||
const identifier = z.string().min(1).max(128);
|
||||
const nullableText = z.string().nullable();
|
||||
const capability = z.enum(["available", "unavailable"]);
|
||||
const restSnapshotSchema = z.object({
|
||||
schemaVersion: z.literal(1),
|
||||
capabilities: z.object({
|
||||
tables: capability,
|
||||
columns: capability,
|
||||
relationships: capability,
|
||||
}).strict(),
|
||||
tables: z.array(z.object({
|
||||
name: identifier,
|
||||
sourceComment: nullableText,
|
||||
}).strict()),
|
||||
columns: z.array(z.object({
|
||||
tableName: identifier,
|
||||
name: identifier,
|
||||
ordinalPosition: z.number().int().positive(),
|
||||
dataType: z.string().min(1).max(2_000),
|
||||
isNullable: z.boolean(),
|
||||
defaultExpression: nullableText,
|
||||
primaryKeyPosition: z.number().int().positive().nullable(),
|
||||
sourceComment: nullableText,
|
||||
}).strict()),
|
||||
relationships: z.array(z.object({
|
||||
constraintName: identifier,
|
||||
sourceTableName: identifier,
|
||||
targetTableName: identifier,
|
||||
updateRule: z.string().min(1).max(64),
|
||||
deleteRule: z.string().min(1).max(64),
|
||||
deferrable: z.boolean(),
|
||||
initiallyDeferred: z.boolean(),
|
||||
columns: z.array(z.object({
|
||||
position: z.number().int().positive(),
|
||||
sourceColumnName: identifier,
|
||||
targetColumnName: identifier,
|
||||
}).strict()).min(1),
|
||||
}).strict()),
|
||||
}).strict();
|
||||
|
||||
function sqlString(value: string): string {
|
||||
return `'${value.replaceAll("'", "''")}'`;
|
||||
}
|
||||
|
||||
function restSnapshotQuery(schemaName: string): string {
|
||||
const schema = sqlString(schemaName);
|
||||
return `WITH target_schema AS (
|
||||
SELECT oid
|
||||
FROM pg_catalog.pg_namespace
|
||||
WHERE nspname = ${schema}
|
||||
),
|
||||
observed_tables AS (
|
||||
SELECT c.oid,
|
||||
c.relname AS name,
|
||||
d.description AS source_comment
|
||||
FROM pg_catalog.pg_class c
|
||||
JOIN target_schema n ON n.oid = c.relnamespace
|
||||
LEFT JOIN pg_catalog.pg_description d ON d.objoid = c.oid AND d.objsubid = 0
|
||||
WHERE c.relkind IN ('r', 'p')
|
||||
),
|
||||
primary_key_columns AS (
|
||||
SELECT i.indrelid AS table_oid,
|
||||
key.attnum,
|
||||
key.ordinality::integer AS position
|
||||
FROM pg_catalog.pg_index i
|
||||
CROSS JOIN LATERAL unnest(i.indkey) WITH ORDINALITY AS key(attnum, ordinality)
|
||||
WHERE i.indisprimary
|
||||
),
|
||||
observed_columns AS (
|
||||
SELECT table_info.name AS table_name,
|
||||
a.attname AS name,
|
||||
a.attnum::integer AS ordinal_position,
|
||||
pg_catalog.format_type(a.atttypid, a.atttypmod) AS data_type,
|
||||
NOT a.attnotnull AS is_nullable,
|
||||
pg_catalog.pg_get_expr(ad.adbin, ad.adrelid) AS default_expression,
|
||||
pk.position AS primary_key_position,
|
||||
d.description AS source_comment
|
||||
FROM observed_tables table_info
|
||||
JOIN pg_catalog.pg_attribute a ON a.attrelid = table_info.oid
|
||||
LEFT JOIN pg_catalog.pg_attrdef ad ON ad.adrelid = table_info.oid AND ad.adnum = a.attnum
|
||||
LEFT JOIN pg_catalog.pg_description d ON d.objoid = table_info.oid AND d.objsubid = a.attnum
|
||||
LEFT JOIN primary_key_columns pk ON pk.table_oid = table_info.oid AND pk.attnum = a.attnum
|
||||
WHERE a.attnum > 0
|
||||
AND NOT a.attisdropped
|
||||
),
|
||||
relationship_pairs AS (
|
||||
SELECT con.oid AS constraint_oid,
|
||||
con.conname AS constraint_name,
|
||||
source_table.relname AS source_table_name,
|
||||
target_table.relname AS target_table_name,
|
||||
CASE con.confupdtype
|
||||
WHEN 'a' THEN 'NO ACTION'
|
||||
WHEN 'r' THEN 'RESTRICT'
|
||||
WHEN 'c' THEN 'CASCADE'
|
||||
WHEN 'n' THEN 'SET NULL'
|
||||
WHEN 'd' THEN 'SET DEFAULT'
|
||||
END AS update_rule,
|
||||
CASE con.confdeltype
|
||||
WHEN 'a' THEN 'NO ACTION'
|
||||
WHEN 'r' THEN 'RESTRICT'
|
||||
WHEN 'c' THEN 'CASCADE'
|
||||
WHEN 'n' THEN 'SET NULL'
|
||||
WHEN 'd' THEN 'SET DEFAULT'
|
||||
END AS delete_rule,
|
||||
con.condeferrable AS is_deferrable,
|
||||
con.condeferred AS initially_deferred,
|
||||
source_key.ordinality::integer AS position,
|
||||
source_column.attname AS source_column_name,
|
||||
target_column.attname AS target_column_name
|
||||
FROM pg_catalog.pg_constraint con
|
||||
JOIN pg_catalog.pg_class source_table ON source_table.oid = con.conrelid
|
||||
JOIN target_schema source_namespace ON source_namespace.oid = source_table.relnamespace
|
||||
JOIN pg_catalog.pg_class target_table ON target_table.oid = con.confrelid
|
||||
JOIN target_schema target_namespace ON target_namespace.oid = target_table.relnamespace
|
||||
JOIN LATERAL unnest(con.conkey) WITH ORDINALITY AS source_key(attnum, ordinality) ON true
|
||||
JOIN LATERAL unnest(con.confkey) WITH ORDINALITY AS target_key(attnum, ordinality)
|
||||
ON target_key.ordinality = source_key.ordinality
|
||||
JOIN pg_catalog.pg_attribute source_column
|
||||
ON source_column.attrelid = source_table.oid AND source_column.attnum = source_key.attnum
|
||||
JOIN pg_catalog.pg_attribute target_column
|
||||
ON target_column.attrelid = target_table.oid AND target_column.attnum = target_key.attnum
|
||||
WHERE con.contype = 'f'
|
||||
),
|
||||
observed_relationships AS (
|
||||
SELECT constraint_oid,
|
||||
constraint_name,
|
||||
source_table_name,
|
||||
target_table_name,
|
||||
update_rule,
|
||||
delete_rule,
|
||||
is_deferrable,
|
||||
initially_deferred,
|
||||
pg_catalog.jsonb_agg(
|
||||
pg_catalog.jsonb_build_object(
|
||||
'position', position,
|
||||
'sourceColumnName', source_column_name,
|
||||
'targetColumnName', target_column_name
|
||||
) ORDER BY position
|
||||
) AS columns
|
||||
FROM relationship_pairs
|
||||
GROUP BY constraint_oid, constraint_name, source_table_name, target_table_name,
|
||||
update_rule, delete_rule, is_deferrable, initially_deferred
|
||||
)
|
||||
SELECT 1 AS "schemaVersion",
|
||||
pg_catalog.jsonb_build_object(
|
||||
'tables', 'available',
|
||||
'columns', 'available',
|
||||
'relationships', 'available'
|
||||
) AS capabilities,
|
||||
COALESCE((
|
||||
SELECT pg_catalog.jsonb_agg(
|
||||
pg_catalog.jsonb_build_object('name', name, 'sourceComment', source_comment)
|
||||
ORDER BY name
|
||||
)
|
||||
FROM observed_tables
|
||||
), '[]'::jsonb) AS tables,
|
||||
COALESCE((
|
||||
SELECT pg_catalog.jsonb_agg(
|
||||
pg_catalog.jsonb_build_object(
|
||||
'tableName', table_name,
|
||||
'name', name,
|
||||
'ordinalPosition', ordinal_position,
|
||||
'dataType', data_type,
|
||||
'isNullable', is_nullable,
|
||||
'defaultExpression', default_expression,
|
||||
'primaryKeyPosition', primary_key_position,
|
||||
'sourceComment', source_comment
|
||||
) ORDER BY table_name, ordinal_position
|
||||
)
|
||||
FROM observed_columns
|
||||
), '[]'::jsonb) AS columns,
|
||||
COALESCE((
|
||||
SELECT pg_catalog.jsonb_agg(
|
||||
pg_catalog.jsonb_build_object(
|
||||
'constraintName', constraint_name,
|
||||
'sourceTableName', source_table_name,
|
||||
'targetTableName', target_table_name,
|
||||
'updateRule', update_rule,
|
||||
'deleteRule', delete_rule,
|
||||
'deferrable', is_deferrable,
|
||||
'initiallyDeferred', initially_deferred,
|
||||
'columns', columns
|
||||
) ORDER BY source_table_name, constraint_name
|
||||
)
|
||||
FROM observed_relationships
|
||||
), '[]'::jsonb) AS relationships
|
||||
FROM target_schema`;
|
||||
}
|
||||
|
||||
function required(value: string | undefined): string {
|
||||
if (!value) throw new CatalogConnectorError("Database binding is incomplete");
|
||||
return value;
|
||||
}
|
||||
|
||||
function textOrNull(value: unknown): string | null {
|
||||
return typeof value === "string" && value.length > 0 ? value : null;
|
||||
}
|
||||
|
||||
function actionRule(value: unknown): string {
|
||||
const rules: Record<string, string> = {
|
||||
a: "NO ACTION",
|
||||
r: "RESTRICT",
|
||||
c: "CASCADE",
|
||||
n: "SET NULL",
|
||||
d: "SET DEFAULT",
|
||||
};
|
||||
const rule = rules[String(value)];
|
||||
if (!rule) throw new CatalogConnectorError("Schema introspection returned an unknown relationship action");
|
||||
return rule;
|
||||
}
|
||||
|
||||
function normalized(snapshot: ObservedSchemaSnapshot): ObservedSchemaSnapshot {
|
||||
const tables = new Map<string, ObservedCatalogTable>();
|
||||
for (const table of snapshot.tables) {
|
||||
if (tables.has(table.name)) throw new CatalogConnectorError("Schema introspection returned duplicate tables");
|
||||
tables.set(table.name, table);
|
||||
}
|
||||
const columns = new Map<string, ObservedCatalogColumn>();
|
||||
for (const column of snapshot.columns) {
|
||||
const key = `${column.tableName}\u0000${column.name}`;
|
||||
if (columns.has(key)) throw new CatalogConnectorError("Schema introspection returned duplicate columns");
|
||||
columns.set(key, column);
|
||||
}
|
||||
const relationships = new Map<string, ObservedCatalogRelationship>();
|
||||
for (const relationship of snapshot.relationships) {
|
||||
const key = `${relationship.sourceTableName}\u0000${relationship.constraintName}`;
|
||||
if (relationships.has(key)) throw new CatalogConnectorError("Schema introspection returned duplicate relationships");
|
||||
relationships.set(key, {
|
||||
...relationship,
|
||||
columns: [...relationship.columns].sort((a, b) => a.position - b.position),
|
||||
});
|
||||
}
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
capabilities: snapshot.capabilities,
|
||||
tables: [...tables.values()].sort((a, b) => a.name.localeCompare(b.name)),
|
||||
columns: [...columns.values()].sort((a, b) => (
|
||||
a.tableName.localeCompare(b.tableName) || a.ordinalPosition - b.ordinalPosition
|
||||
)),
|
||||
relationships: [...relationships.values()].sort((a, b) => (
|
||||
a.sourceTableName.localeCompare(b.sourceTableName) || a.constraintName.localeCompare(b.constraintName)
|
||||
)),
|
||||
};
|
||||
}
|
||||
|
||||
export class ConcreteCatalogSchemaIntrospector implements CatalogSchemaIntrospector {
|
||||
constructor(
|
||||
private readonly postgres: CatalogPostgresAccess,
|
||||
private readonly secretStore: WorkspaceSecretStore,
|
||||
) {}
|
||||
|
||||
async scan(
|
||||
database: WorkspaceDatabase,
|
||||
signal: AbortSignal,
|
||||
progress?: CatalogSchemaScanProgress,
|
||||
): Promise<ObservedSchemaSnapshot> {
|
||||
return database.binding.transport === "rest_api"
|
||||
? await this.scanRest(database, signal, progress)
|
||||
: await this.scanPostgres(database, signal, progress);
|
||||
}
|
||||
|
||||
private async scanPostgres(
|
||||
database: WorkspaceDatabase,
|
||||
signal: AbortSignal,
|
||||
progress?: CatalogSchemaScanProgress,
|
||||
): Promise<ObservedSchemaSnapshot> {
|
||||
await progress?.("connecting");
|
||||
const client = await this.postgres.connect(database, signal);
|
||||
try {
|
||||
const schema = await client.query(
|
||||
"SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname = $1) AS present",
|
||||
[database.schema],
|
||||
);
|
||||
if (schema.rows[0]?.present !== true) throw new CatalogConnectorError("Database schema is unavailable");
|
||||
|
||||
await progress?.("scanning_tables");
|
||||
const tableResult = await client.query(
|
||||
`SELECT c.relname AS name, d.description AS source_comment
|
||||
FROM pg_catalog.pg_class c
|
||||
JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
|
||||
LEFT JOIN pg_catalog.pg_description d ON d.objoid = c.oid AND d.objsubid = 0
|
||||
WHERE c.relkind IN ('r', 'p') AND n.nspname = $1
|
||||
ORDER BY c.relname`,
|
||||
[database.schema],
|
||||
);
|
||||
const tables: ObservedCatalogTable[] = tableResult.rows.map((row) => ({
|
||||
name: String(row.name),
|
||||
sourceComment: textOrNull(row.source_comment),
|
||||
}));
|
||||
await progress?.("scanning_tables", { tables: tables.length });
|
||||
|
||||
await progress?.("scanning_columns", { tables: tables.length });
|
||||
const columnResult = await client.query(
|
||||
`SELECT c.relname AS table_name,
|
||||
a.attname AS name,
|
||||
a.attnum::integer AS ordinal_position,
|
||||
pg_catalog.format_type(a.atttypid, a.atttypmod) AS data_type,
|
||||
NOT a.attnotnull AS is_nullable,
|
||||
pg_catalog.pg_get_expr(ad.adbin, ad.adrelid) AS default_expression,
|
||||
pk.position AS primary_key_position,
|
||||
d.description AS source_comment
|
||||
FROM pg_catalog.pg_class c
|
||||
JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
|
||||
JOIN pg_catalog.pg_attribute a ON a.attrelid = c.oid
|
||||
LEFT JOIN pg_catalog.pg_attrdef ad ON ad.adrelid = c.oid AND ad.adnum = a.attnum
|
||||
LEFT JOIN pg_catalog.pg_description d ON d.objoid = c.oid AND d.objsubid = a.attnum
|
||||
LEFT JOIN LATERAL (
|
||||
SELECT key.ordinality::integer AS position
|
||||
FROM pg_catalog.pg_index i
|
||||
CROSS JOIN LATERAL unnest(i.indkey) WITH ORDINALITY AS key(attnum, ordinality)
|
||||
WHERE i.indrelid = c.oid AND i.indisprimary AND key.attnum = a.attnum
|
||||
LIMIT 1
|
||||
) pk ON true
|
||||
WHERE c.relkind IN ('r', 'p')
|
||||
AND n.nspname = $1
|
||||
AND a.attnum > 0
|
||||
AND NOT a.attisdropped
|
||||
ORDER BY c.relname, a.attnum`,
|
||||
[database.schema],
|
||||
);
|
||||
const columns: ObservedCatalogColumn[] = columnResult.rows.map((row) => ({
|
||||
tableName: String(row.table_name),
|
||||
name: String(row.name),
|
||||
ordinalPosition: Number(row.ordinal_position),
|
||||
dataType: String(row.data_type),
|
||||
isNullable: row.is_nullable === true,
|
||||
defaultExpression: textOrNull(row.default_expression),
|
||||
primaryKeyPosition: row.primary_key_position === null || row.primary_key_position === undefined
|
||||
? null
|
||||
: Number(row.primary_key_position),
|
||||
sourceComment: textOrNull(row.source_comment),
|
||||
}));
|
||||
await progress?.("scanning_columns", { tables: tables.length, columns: columns.length });
|
||||
|
||||
await progress?.("scanning_relationships", { tables: tables.length, columns: columns.length });
|
||||
const relationshipResult = await client.query(
|
||||
`SELECT con.conname AS constraint_name,
|
||||
source_table.relname AS source_table_name,
|
||||
target_table.relname AS target_table_name,
|
||||
con.confupdtype AS update_action,
|
||||
con.confdeltype AS delete_action,
|
||||
con.condeferrable AS deferrable,
|
||||
con.condeferred AS initially_deferred,
|
||||
source_key.ordinality::integer AS position,
|
||||
source_column.attname AS source_column_name,
|
||||
target_column.attname AS target_column_name
|
||||
FROM pg_catalog.pg_constraint con
|
||||
JOIN pg_catalog.pg_class source_table ON source_table.oid = con.conrelid
|
||||
JOIN pg_catalog.pg_namespace source_namespace ON source_namespace.oid = source_table.relnamespace
|
||||
JOIN pg_catalog.pg_class target_table ON target_table.oid = con.confrelid
|
||||
JOIN pg_catalog.pg_namespace target_namespace ON target_namespace.oid = target_table.relnamespace
|
||||
JOIN LATERAL unnest(con.conkey) WITH ORDINALITY AS source_key(attnum, ordinality) ON true
|
||||
JOIN LATERAL unnest(con.confkey) WITH ORDINALITY AS target_key(attnum, ordinality)
|
||||
ON target_key.ordinality = source_key.ordinality
|
||||
JOIN pg_catalog.pg_attribute source_column
|
||||
ON source_column.attrelid = source_table.oid AND source_column.attnum = source_key.attnum
|
||||
JOIN pg_catalog.pg_attribute target_column
|
||||
ON target_column.attrelid = target_table.oid AND target_column.attnum = target_key.attnum
|
||||
WHERE con.contype = 'f'
|
||||
AND source_namespace.nspname = $1
|
||||
AND target_namespace.nspname = $1
|
||||
ORDER BY source_table.relname, con.conname, source_key.ordinality`,
|
||||
[database.schema],
|
||||
);
|
||||
const relationshipMap = new Map<string, ObservedCatalogRelationship>();
|
||||
for (const row of relationshipResult.rows) {
|
||||
const sourceTableName = String(row.source_table_name);
|
||||
const constraintName = String(row.constraint_name);
|
||||
const key = `${sourceTableName}\u0000${constraintName}`;
|
||||
const current = relationshipMap.get(key) ?? {
|
||||
constraintName,
|
||||
sourceTableName,
|
||||
targetTableName: String(row.target_table_name),
|
||||
updateRule: actionRule(row.update_action),
|
||||
deleteRule: actionRule(row.delete_action),
|
||||
deferrable: row.deferrable === true,
|
||||
initiallyDeferred: row.initially_deferred === true,
|
||||
columns: [],
|
||||
};
|
||||
current.columns.push({
|
||||
position: Number(row.position),
|
||||
sourceColumnName: String(row.source_column_name),
|
||||
targetColumnName: String(row.target_column_name),
|
||||
});
|
||||
relationshipMap.set(key, current);
|
||||
}
|
||||
const relationships = [...relationshipMap.values()];
|
||||
await progress?.("scanning_relationships", {
|
||||
tables: tables.length,
|
||||
columns: columns.length,
|
||||
relationships: relationships.length,
|
||||
});
|
||||
return normalized({
|
||||
schemaVersion: 1,
|
||||
capabilities: { tables: "available", columns: "available", relationships: "available" },
|
||||
tables,
|
||||
columns,
|
||||
relationships,
|
||||
});
|
||||
} finally {
|
||||
await client.end();
|
||||
}
|
||||
}
|
||||
|
||||
private async scanRest(
|
||||
database: WorkspaceDatabase,
|
||||
signal: AbortSignal,
|
||||
progress?: CatalogSchemaScanProgress,
|
||||
): Promise<ObservedSchemaSnapshot> {
|
||||
await progress?.("connecting");
|
||||
const auth = database.binding.restAuth ?? "bearer";
|
||||
const materialized = this.secretStore.materialize(
|
||||
database.workspaceId,
|
||||
auth === "none" ? [] : [CATALOG_SECRET_IDS.apiKey],
|
||||
);
|
||||
try {
|
||||
const headers: Record<string, string> = { "content-type": "application/json" };
|
||||
if (auth !== "none") {
|
||||
const credentialFile = materialized.files.get(CATALOG_SECRET_IDS.apiKey);
|
||||
if (!credentialFile) throw new CatalogConnectorError("REST API key is not configured");
|
||||
const credential = (await readFile(credentialFile, "utf8")).trim();
|
||||
if (auth === "bearer") headers.authorization = `Bearer ${credential}`;
|
||||
else headers["x-api-key"] = credential;
|
||||
}
|
||||
const baseUrl = required(database.binding.baseUrl).replace(/\/+$/, "");
|
||||
const response = await fetch(`${baseUrl}/rpc/schema_snapshot`, {
|
||||
method: "POST",
|
||||
headers,
|
||||
body: JSON.stringify({ schema_name: database.schema }),
|
||||
signal,
|
||||
});
|
||||
let body: unknown;
|
||||
if (response.ok) {
|
||||
body = await response.json();
|
||||
} else if (response.status === 404) {
|
||||
const fallback = await fetch(`${baseUrl}/rpc/run_query`, {
|
||||
method: "POST",
|
||||
headers,
|
||||
body: JSON.stringify({ query_text: restSnapshotQuery(database.schema) }),
|
||||
signal,
|
||||
});
|
||||
if (!fallback.ok) throw new CatalogSchemaCapabilityUnavailableError("schema_snapshot");
|
||||
const rows: unknown = await fallback.json();
|
||||
if (!Array.isArray(rows) || rows.length !== 1) {
|
||||
throw new CatalogConnectorError("REST schema snapshot fallback is invalid");
|
||||
}
|
||||
body = rows[0];
|
||||
} else {
|
||||
throw new CatalogSchemaCapabilityUnavailableError("schema_snapshot");
|
||||
}
|
||||
const parsed = restSnapshotSchema.safeParse(body);
|
||||
if (!parsed.success) throw new CatalogConnectorError("REST schema snapshot is invalid");
|
||||
const snapshot = normalized(parsed.data);
|
||||
await progress?.("scanning_tables", { tables: snapshot.tables.length });
|
||||
await progress?.("scanning_columns", { tables: snapshot.tables.length, columns: snapshot.columns.length });
|
||||
await progress?.("scanning_relationships", {
|
||||
tables: snapshot.tables.length,
|
||||
columns: snapshot.columns.length,
|
||||
relationships: snapshot.relationships.length,
|
||||
});
|
||||
return snapshot;
|
||||
} catch (error) {
|
||||
if (error instanceof CatalogConnectorError) throw error;
|
||||
throw new CatalogConnectorError("REST schema introspection failed");
|
||||
} finally {
|
||||
materialized.release();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
export const CATALOG_SECRET_IDS = {
|
||||
password: "catalog.dwh.password",
|
||||
apiKey: "catalog.dwh.api_key",
|
||||
sshPrivateKey: "catalog.dwh.ssh_private_key",
|
||||
sshPrivateKeyPassphrase: "catalog.dwh.ssh_private_key_passphrase",
|
||||
sshKnownHosts: "catalog.dwh.ssh_known_hosts",
|
||||
tlsCa: "catalog.dwh.tls_ca",
|
||||
} as const;
|
||||
|
||||
export type CatalogSecretName = keyof typeof CATALOG_SECRET_IDS;
|
||||
@@ -0,0 +1,228 @@
|
||||
import { buildInstallationContract } from "../workspaces/contracts.js";
|
||||
import { resolveBinding } from "../workspaces/bindings.js";
|
||||
import type { WorkspaceRegistry } from "../workspaces/registry.js";
|
||||
import type { WorkspaceDescriptor } from "../workspaces/schema.js";
|
||||
import type { WorkspaceSecretStore } from "../workspaces/secret-store.js";
|
||||
import { createConcreteDiagnosticAdapters } from "../workspaces/diagnostics.js";
|
||||
import { CatalogOperationCoordinator } from "./operation-coordinator.js";
|
||||
import {
|
||||
ConcreteCatalogPostgresAccess,
|
||||
type CatalogPostgresAccess,
|
||||
} from "./postgres-access.js";
|
||||
import type {
|
||||
CatalogRepository,
|
||||
DatabaseBinding,
|
||||
DatabaseConfigurationInput,
|
||||
DatabaseTestResult,
|
||||
WorkspaceDatabase,
|
||||
} from "./types.js";
|
||||
import { CATALOG_SECRET_IDS, type CatalogSecretName } from "./secrets.js";
|
||||
|
||||
export { CATALOG_SECRET_IDS, type CatalogSecretName } from "./secrets.js";
|
||||
|
||||
export interface CatalogListItem extends Omit<WorkspaceDatabase, "id"> {
|
||||
id?: string;
|
||||
workspaceName: string;
|
||||
workspaceDescription?: string;
|
||||
workspaceAvailable: boolean;
|
||||
configured: boolean;
|
||||
secrets: Record<CatalogSecretName, boolean>;
|
||||
}
|
||||
|
||||
function bindingValue(workspace: WorkspaceDescriptor, values: Record<string, string>, suffix: string) {
|
||||
const variable = buildInstallationContract(workspace).variables.find((entry) => (
|
||||
entry.role === "DWH" && entry.suffix === suffix
|
||||
));
|
||||
return variable ? values[variable.name] : undefined;
|
||||
}
|
||||
|
||||
function numeric(value: string | undefined): number | undefined {
|
||||
if (!value) return undefined;
|
||||
const parsed = Number(value);
|
||||
return Number.isInteger(parsed) && parsed >= 1 && parsed <= 65_535 ? parsed : undefined;
|
||||
}
|
||||
|
||||
function yamlBinding(workspace: WorkspaceDescriptor, secretRoots: readonly string[]): DatabaseBinding {
|
||||
const effective = resolveBinding(workspace, "DWH", process.env, secretRoots);
|
||||
const value = (suffix: string) => bindingValue(workspace, effective.values, suffix);
|
||||
return {
|
||||
transport: effective.transport,
|
||||
host: value("HOST"),
|
||||
port: numeric(value("PORT")) ?? workspace.dwh.port,
|
||||
username: value("USER"),
|
||||
baseUrl: value("BASE_URL"),
|
||||
restPath: workspace.diagnostics?.dwh_rest?.path ?? "/health",
|
||||
restAuth: workspace.diagnostics?.dwh_rest?.auth ?? "bearer",
|
||||
tlsServername: value("TLS_SERVERNAME"),
|
||||
sshHost: value("SSH_HOST"),
|
||||
sshPort: numeric(value("SSH_PORT")),
|
||||
sshUsername: value("SSH_USER"),
|
||||
sshTargetHost: value("SSH_TARGET_HOST"),
|
||||
sshTargetPort: numeric(value("SSH_TARGET_PORT")),
|
||||
};
|
||||
}
|
||||
|
||||
function secretState(store: WorkspaceSecretStore, workspaceId: string): Record<CatalogSecretName, boolean> {
|
||||
return Object.fromEntries(Object.entries(CATALOG_SECRET_IDS).map(([name, id]) => (
|
||||
[name, store.has(workspaceId, id)]
|
||||
))) as Record<CatalogSecretName, boolean>;
|
||||
}
|
||||
|
||||
export class CatalogService {
|
||||
private readonly adapters = createConcreteDiagnosticAdapters();
|
||||
|
||||
constructor(
|
||||
private readonly repository: CatalogRepository,
|
||||
private readonly registry: WorkspaceRegistry,
|
||||
private readonly secretStore: WorkspaceSecretStore,
|
||||
private readonly secretRoots: readonly string[],
|
||||
private readonly diagnosticTimeoutMs: number,
|
||||
private readonly postgres: CatalogPostgresAccess = new ConcreteCatalogPostgresAccess(secretStore, {
|
||||
connectTimeoutMs: diagnosticTimeoutMs,
|
||||
}),
|
||||
private readonly operations: CatalogOperationCoordinator = new CatalogOperationCoordinator(),
|
||||
) {}
|
||||
|
||||
async list(): Promise<CatalogListItem[]> {
|
||||
const [workspaces, configured] = await Promise.all([
|
||||
this.registry.listCatalog(),
|
||||
this.repository.list(),
|
||||
]);
|
||||
const byWorkspace = new Map(configured.map((database) => [database.workspaceId, database]));
|
||||
const active = await Promise.all(workspaces.map(async (entry) => {
|
||||
const database = byWorkspace.get(entry.id);
|
||||
const { workspace } = await this.registry.read(entry.id);
|
||||
const base = database ?? {
|
||||
workspaceId: entry.id,
|
||||
engine: "postgres" as const,
|
||||
databaseName: workspace.dwh.database,
|
||||
schema: workspace.dwh.schema,
|
||||
version: 0,
|
||||
createdAt: "",
|
||||
updatedAt: "",
|
||||
binding: yamlBinding(workspace, this.secretRoots),
|
||||
connectionStatus: "untested" as const,
|
||||
};
|
||||
return {
|
||||
...base,
|
||||
workspaceName: entry.name,
|
||||
workspaceDescription: entry.description,
|
||||
workspaceAvailable: true,
|
||||
configured: database !== undefined,
|
||||
secrets: secretState(this.secretStore, entry.id),
|
||||
};
|
||||
}));
|
||||
const known = new Set(workspaces.map((entry) => entry.id));
|
||||
const orphaned: CatalogListItem[] = configured
|
||||
.filter((database) => !known.has(database.workspaceId))
|
||||
.map((database) => ({
|
||||
...database,
|
||||
workspaceName: database.workspaceId,
|
||||
workspaceDescription: "Workspace is no longer present in the repository catalog.",
|
||||
workspaceAvailable: false,
|
||||
configured: true,
|
||||
secrets: secretState(this.secretStore, database.workspaceId),
|
||||
}));
|
||||
return [...active, ...orphaned];
|
||||
}
|
||||
|
||||
async ensureWorkspace(workspaceId: string): Promise<WorkspaceDescriptor> {
|
||||
const { workspace } = await this.registry.read(workspaceId);
|
||||
return workspace;
|
||||
}
|
||||
|
||||
async normalizeInput(input: DatabaseConfigurationInput): Promise<DatabaseConfigurationInput> {
|
||||
const workspace = await this.ensureWorkspace(input.workspaceId);
|
||||
if (input.binding.transport !== "rest_api") return input;
|
||||
return {
|
||||
...input,
|
||||
binding: {
|
||||
...input.binding,
|
||||
restPath: workspace.diagnostics?.dwh_rest?.path ?? "/health",
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
configuredSecrets(workspaceId: string): Record<CatalogSecretName, boolean> {
|
||||
return secretState(this.secretStore, workspaceId);
|
||||
}
|
||||
|
||||
replaceSecrets(workspaceId: string, values: Partial<Record<CatalogSecretName, string>>): void {
|
||||
const encoded: Record<string, string> = {};
|
||||
for (const [name, value] of Object.entries(values) as Array<[CatalogSecretName, string | undefined]>) {
|
||||
if (value !== undefined && value.length > 0) encoded[CATALOG_SECRET_IDS[name]] = value;
|
||||
}
|
||||
if (Object.keys(encoded).length > 0) this.secretStore.putMany(workspaceId, encoded);
|
||||
}
|
||||
|
||||
forgetSecrets(workspaceId: string): void {
|
||||
for (const id of Object.values(CATALOG_SECRET_IDS)) this.secretStore.forget(workspaceId, id);
|
||||
}
|
||||
|
||||
async test(database: WorkspaceDatabase): Promise<DatabaseTestResult> {
|
||||
return await this.operations.run(database.id, async () => {
|
||||
const testedAt = new Date().toISOString();
|
||||
const controller = new AbortController();
|
||||
const timer = setTimeout(() => controller.abort(), this.diagnosticTimeoutMs);
|
||||
const required = database.binding.transport === "rest_api"
|
||||
? database.binding.restAuth === "none" ? [] : [CATALOG_SECRET_IDS.apiKey]
|
||||
: [];
|
||||
const materialized = this.secretStore.materialize(database.workspaceId, required);
|
||||
try {
|
||||
if (database.binding.transport !== "rest_api") {
|
||||
const client = await this.postgres.connect(database, controller.signal);
|
||||
try {
|
||||
const result = await client.query(
|
||||
`SELECT current_database() AS database,
|
||||
CASE WHEN pg_catalog.has_schema_privilege(
|
||||
current_user,
|
||||
(SELECT oid FROM pg_catalog.pg_namespace WHERE nspname = $1),
|
||||
'USAGE'
|
||||
) THEN $1 ELSE NULL END AS schema`,
|
||||
[database.schema],
|
||||
);
|
||||
const row = result.rows[0];
|
||||
if (row?.database !== database.databaseName || row.schema !== database.schema) {
|
||||
throw new Error("Database identity mismatch");
|
||||
}
|
||||
} finally {
|
||||
await client.end();
|
||||
}
|
||||
} else {
|
||||
const credentialId = CATALOG_SECRET_IDS.apiKey;
|
||||
await this.adapters.probeConnector({
|
||||
role: "dwh",
|
||||
transport: database.binding.transport,
|
||||
baseUrl: database.binding.baseUrl,
|
||||
credentialFile: materialized.files.get(credentialId),
|
||||
resource: { database: database.databaseName, schema: database.schema },
|
||||
timeoutMs: this.diagnosticTimeoutMs,
|
||||
signal: controller.signal,
|
||||
diagnostic: {
|
||||
method: "GET" as const,
|
||||
path: database.binding.restPath ?? "/health",
|
||||
auth: database.binding.restAuth ?? "bearer",
|
||||
},
|
||||
});
|
||||
}
|
||||
return {
|
||||
connectionStatus: "reachable",
|
||||
testedVersion: database.version,
|
||||
lastTestedAt: testedAt,
|
||||
};
|
||||
} catch {
|
||||
return {
|
||||
connectionStatus: "failed",
|
||||
testedVersion: database.version,
|
||||
lastTestedAt: testedAt,
|
||||
errorCode: "connector_unavailable",
|
||||
errorMessage: "The database connector could not be reached or authenticated.",
|
||||
};
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
controller.abort();
|
||||
materialized.release();
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,311 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import type { CatalogOperationCoordinator } from "./operation-coordinator.js";
|
||||
import type { CatalogSchemaIntrospector, CatalogSchemaScanProgress } from "./schema-introspector.js";
|
||||
import {
|
||||
CatalogConflictError,
|
||||
CatalogConnectorError,
|
||||
CatalogSchemaCapabilityUnavailableError,
|
||||
type CatalogRepository,
|
||||
type CatalogSchemaDiff,
|
||||
type CatalogSyncCounts,
|
||||
type CatalogSyncRun,
|
||||
type CatalogSyncScope,
|
||||
type ObservedSchemaSnapshot,
|
||||
type WorkspaceDatabase,
|
||||
} from "./types.js";
|
||||
|
||||
class SyncCancelledError extends Error {}
|
||||
|
||||
const TERMINAL_STATES = new Set<CatalogSyncRun["state"]>([
|
||||
"succeeded", "failed", "cancelled", "interrupted",
|
||||
]);
|
||||
|
||||
function destructive(diff: CatalogSchemaDiff): boolean {
|
||||
return diff.deletedTables.length > 0
|
||||
|| diff.deletedColumns.length > 0
|
||||
|| diff.deletedRelationships.length > 0;
|
||||
}
|
||||
|
||||
function fingerprint(snapshot: ObservedSchemaSnapshot): string {
|
||||
return JSON.stringify(snapshot);
|
||||
}
|
||||
|
||||
function safeFailure(error: unknown): { code: string; message: string } {
|
||||
if (error instanceof CatalogSchemaCapabilityUnavailableError) {
|
||||
const label = error.capability === "schema_snapshot" ? "schema snapshot" : error.capability;
|
||||
return {
|
||||
code: "schema_capability_unavailable",
|
||||
message: `This database binding does not provide the ${label} capability.`,
|
||||
};
|
||||
}
|
||||
if (error instanceof CatalogConnectorError) {
|
||||
return { code: "schema_introspection_failed", message: "The database schema could not be read safely." };
|
||||
}
|
||||
return { code: "schema_sync_failed", message: "Schema synchronization failed." };
|
||||
}
|
||||
|
||||
export class CatalogSyncWorker {
|
||||
private readonly workerId = randomUUID();
|
||||
private readonly controllers = new Map<string, AbortController>();
|
||||
private readonly reservations = new Map<string, () => void>();
|
||||
private stopping = false;
|
||||
|
||||
constructor(
|
||||
private readonly repository: CatalogRepository,
|
||||
private readonly introspector: CatalogSchemaIntrospector,
|
||||
private readonly operations: CatalogOperationCoordinator,
|
||||
private readonly timeoutMs: number,
|
||||
) {}
|
||||
|
||||
async initialize(): Promise<void> {
|
||||
if (!(await this.repository.available())) return;
|
||||
await this.repository.interruptActiveSyncRuns();
|
||||
await this.repository.pruneSyncEvents(new Date(Date.now() - 30 * 24 * 60 * 60 * 1_000).toISOString());
|
||||
}
|
||||
|
||||
async start(database: WorkspaceDatabase, scope: CatalogSyncScope, tableIds: readonly string[]): Promise<CatalogSyncRun> {
|
||||
this.assertReady(database);
|
||||
const uniqueTableIds = [...new Set(tableIds)];
|
||||
if (scope === "columns") {
|
||||
const tables = await Promise.all(uniqueTableIds.map((tableId) => this.repository.getTable(database.id, tableId)));
|
||||
if (tables.some((table) => !table)) throw new CatalogConflictError("One or more selected tables no longer exist");
|
||||
}
|
||||
if (scope !== "columns" && uniqueTableIds.length > 0) {
|
||||
throw new CatalogConflictError("Table selection is only valid for a column synchronization");
|
||||
}
|
||||
const release = this.operations.reserve(database.id);
|
||||
try {
|
||||
const run = await this.repository.createSyncRun(database.id, scope, uniqueTableIds, database.version);
|
||||
this.reservations.set(run.id, release);
|
||||
await this.repository.appendSyncEvent(run.id, "info", "queued", "Synchronization queued.", { scope });
|
||||
this.launch(run.id);
|
||||
return run;
|
||||
} catch (error) {
|
||||
release();
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async confirm(runId: string, confirmationToken: string): Promise<CatalogSyncRun | undefined> {
|
||||
const run = await this.repository.getSyncRun(runId);
|
||||
if (!run) return undefined;
|
||||
if (run.state !== "awaiting_confirmation" || !run.observedSnapshot || run.confirmationToken !== confirmationToken) {
|
||||
throw new CatalogConflictError("Synchronization confirmation is no longer valid");
|
||||
}
|
||||
await this.repository.appendSyncEvent(run.id, "info", "confirmation_received", "Destructive changes were confirmed.");
|
||||
const queued = await this.repository.updateSyncRun(run.id, {
|
||||
state: "queued",
|
||||
phase: "queued",
|
||||
confirmationToken: null,
|
||||
leaseOwner: null,
|
||||
leaseExpiresAt: null,
|
||||
});
|
||||
this.launch(run.id, fingerprint(run.observedSnapshot));
|
||||
return queued;
|
||||
}
|
||||
|
||||
async cancel(runId: string): Promise<CatalogSyncRun | undefined> {
|
||||
const run = await this.repository.getSyncRun(runId);
|
||||
if (!run) return undefined;
|
||||
if (run.state === "applying" || TERMINAL_STATES.has(run.state)) return run;
|
||||
await this.repository.requestSyncRunCancellation(runId);
|
||||
this.controllers.get(runId)?.abort();
|
||||
if (run.state === "queued" || run.state === "awaiting_confirmation") {
|
||||
const cancelled = await this.repository.updateSyncRun(runId, {
|
||||
state: "cancelled",
|
||||
phase: "completed",
|
||||
finishedAt: new Date().toISOString(),
|
||||
observedSnapshot: null,
|
||||
plannedDiff: null,
|
||||
confirmationToken: null,
|
||||
leaseOwner: null,
|
||||
leaseExpiresAt: null,
|
||||
});
|
||||
await this.repository.appendSyncEvent(runId, "warning", "cancelled", "Synchronization cancelled.");
|
||||
this.release(runId);
|
||||
return cancelled;
|
||||
}
|
||||
return await this.repository.getSyncRun(runId);
|
||||
}
|
||||
|
||||
async retry(runId: string): Promise<CatalogSyncRun | undefined> {
|
||||
const previous = await this.repository.getSyncRun(runId);
|
||||
if (!previous) return undefined;
|
||||
if (!TERMINAL_STATES.has(previous.state)) {
|
||||
throw new CatalogConflictError("Only a finished synchronization can be retried");
|
||||
}
|
||||
const database = await this.repository.get(previous.databaseId);
|
||||
if (!database) return undefined;
|
||||
return await this.start(database, previous.scope, previous.tableIds);
|
||||
}
|
||||
|
||||
async stop(): Promise<void> {
|
||||
this.stopping = true;
|
||||
for (const controller of this.controllers.values()) controller.abort();
|
||||
if (await this.repository.available()) await this.repository.interruptActiveSyncRuns();
|
||||
for (const runId of [...this.reservations.keys()]) this.release(runId);
|
||||
}
|
||||
|
||||
private launch(runId: string, confirmedFingerprint?: string): void {
|
||||
queueMicrotask(() => {
|
||||
void this.execute(runId, confirmedFingerprint).catch(() => undefined);
|
||||
});
|
||||
}
|
||||
|
||||
private async execute(runId: string, confirmedFingerprint?: string): Promise<void> {
|
||||
if (this.stopping) return;
|
||||
const leaseExpiresAt = new Date(Date.now() + 20_000).toISOString();
|
||||
const claimed = await this.repository.claimSyncRun(runId, this.workerId, leaseExpiresAt);
|
||||
if (!claimed) return;
|
||||
const controller = new AbortController();
|
||||
this.controllers.set(runId, controller);
|
||||
let timedOut = false;
|
||||
const timeout = setTimeout(() => {
|
||||
timedOut = true;
|
||||
controller.abort();
|
||||
}, this.timeoutMs);
|
||||
const heartbeat = setInterval(() => {
|
||||
void this.repository.updateSyncRun(runId, {
|
||||
heartbeatAt: new Date().toISOString(),
|
||||
leaseExpiresAt: new Date(Date.now() + 20_000).toISOString(),
|
||||
});
|
||||
}, 5_000);
|
||||
|
||||
try {
|
||||
await this.repository.appendSyncEvent(runId, "info", "started", "Synchronization started.");
|
||||
const database = await this.repository.get(claimed.databaseId);
|
||||
if (!database || database.version !== claimed.requestedDatabaseVersion) {
|
||||
throw new CatalogConflictError("Database binding changed before synchronization started");
|
||||
}
|
||||
this.assertReady(database);
|
||||
const progress: CatalogSchemaScanProgress = async (phase, counts) => {
|
||||
await this.checkCancelled(runId);
|
||||
await this.repository.updateSyncRun(runId, {
|
||||
phase,
|
||||
heartbeatAt: new Date().toISOString(),
|
||||
...(counts ? { counts } : {}),
|
||||
});
|
||||
await this.repository.appendSyncEvent(runId, "info", phase, this.phaseMessage(phase), counts ?? {});
|
||||
};
|
||||
const snapshot = await this.introspector.scan(database, controller.signal, progress);
|
||||
await this.checkCancelled(runId);
|
||||
this.assertCapability(claimed.scope, snapshot);
|
||||
const counts: CatalogSyncCounts = {
|
||||
tables: snapshot.tables.length,
|
||||
columns: snapshot.columns.length,
|
||||
relationships: snapshot.relationships.length,
|
||||
};
|
||||
await this.repository.updateSyncRun(runId, { phase: "planning", counts, observedSnapshot: snapshot });
|
||||
await this.repository.appendSyncEvent(runId, "info", "planning", "Schema changes are being planned.", { ...counts });
|
||||
const diff = await this.repository.planSchemaSync(claimed.databaseId, claimed.scope, claimed.tableIds, snapshot);
|
||||
await this.checkCancelled(runId);
|
||||
if (destructive(diff) && fingerprint(snapshot) !== confirmedFingerprint) {
|
||||
const token = randomUUID();
|
||||
const waiting = await this.repository.updateSyncRun(runId, {
|
||||
state: "awaiting_confirmation",
|
||||
phase: "awaiting_confirmation",
|
||||
observedSnapshot: snapshot,
|
||||
plannedDiff: diff,
|
||||
confirmationToken: token,
|
||||
counts,
|
||||
leaseOwner: null,
|
||||
leaseExpiresAt: null,
|
||||
});
|
||||
await this.repository.appendSyncEvent(runId, "warning", "confirmation_required", "Confirmation is required before removing catalog objects.", {
|
||||
deletedTables: diff.deletedTables.length,
|
||||
deletedColumns: diff.deletedColumns.length,
|
||||
deletedRelationships: diff.deletedRelationships.length,
|
||||
});
|
||||
if (!waiting) throw new Error("Synchronization run disappeared");
|
||||
return;
|
||||
}
|
||||
|
||||
await this.repository.updateSyncRun(runId, { state: "applying", phase: "applying", plannedDiff: diff });
|
||||
await this.repository.appendSyncEvent(runId, "info", "applying", "Catalog changes are being applied atomically.");
|
||||
this.controllers.delete(runId);
|
||||
const applied = await this.repository.applySchemaSync(
|
||||
claimed.databaseId,
|
||||
claimed.requestedDatabaseVersion,
|
||||
claimed.scope,
|
||||
claimed.tableIds,
|
||||
snapshot,
|
||||
);
|
||||
if (!applied) throw new CatalogConflictError("Database binding changed before schema changes were applied");
|
||||
await this.repository.updateSyncRun(runId, {
|
||||
state: "succeeded",
|
||||
phase: "completed",
|
||||
counts: applied,
|
||||
finishedAt: new Date().toISOString(),
|
||||
observedSnapshot: null,
|
||||
plannedDiff: null,
|
||||
confirmationToken: null,
|
||||
heartbeatAt: new Date().toISOString(),
|
||||
leaseOwner: null,
|
||||
leaseExpiresAt: null,
|
||||
});
|
||||
await this.repository.appendSyncEvent(runId, "info", "succeeded", "Synchronization completed.", { ...applied });
|
||||
this.release(runId);
|
||||
} catch (error) {
|
||||
const current = await this.repository.getSyncRun(runId);
|
||||
const cancelled = !timedOut && (error instanceof SyncCancelledError || controller.signal.aborted || current?.cancelRequested);
|
||||
const failure = timedOut
|
||||
? { code: "schema_sync_timed_out", message: "Schema synchronization timed out." }
|
||||
: safeFailure(error);
|
||||
await this.repository.updateSyncRun(runId, {
|
||||
state: cancelled ? "cancelled" : "failed",
|
||||
phase: "completed",
|
||||
errorCode: cancelled ? null : failure.code,
|
||||
errorMessage: cancelled ? null : failure.message,
|
||||
finishedAt: new Date().toISOString(),
|
||||
observedSnapshot: null,
|
||||
plannedDiff: null,
|
||||
confirmationToken: null,
|
||||
leaseOwner: null,
|
||||
leaseExpiresAt: null,
|
||||
});
|
||||
await this.repository.appendSyncEvent(
|
||||
runId,
|
||||
cancelled ? "warning" : "error",
|
||||
cancelled ? "cancelled" : "failed",
|
||||
cancelled ? "Synchronization cancelled." : failure.message,
|
||||
);
|
||||
this.release(runId);
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
clearInterval(heartbeat);
|
||||
this.controllers.delete(runId);
|
||||
}
|
||||
}
|
||||
|
||||
private assertReady(database: WorkspaceDatabase): void {
|
||||
if (database.connectionStatus !== "reachable" || database.testedVersion !== database.version) {
|
||||
throw new CatalogConflictError("Test the current database binding before synchronizing its schema");
|
||||
}
|
||||
}
|
||||
|
||||
private assertCapability(scope: CatalogSyncScope, snapshot: ObservedSchemaSnapshot): void {
|
||||
const required = scope === "all" ? ["tables", "columns", "relationships"] as const : [scope] as const;
|
||||
for (const name of required) {
|
||||
if (snapshot.capabilities[name] !== "available") {
|
||||
throw new CatalogSchemaCapabilityUnavailableError(name);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async checkCancelled(runId: string): Promise<void> {
|
||||
const run = await this.repository.getSyncRun(runId);
|
||||
if (run?.cancelRequested) throw new SyncCancelledError("Synchronization cancelled");
|
||||
}
|
||||
|
||||
private release(runId: string): void {
|
||||
this.reservations.get(runId)?.();
|
||||
this.reservations.delete(runId);
|
||||
}
|
||||
|
||||
private phaseMessage(phase: Parameters<CatalogSchemaScanProgress>[0]): string {
|
||||
if (phase === "connecting") return "Connecting to the database.";
|
||||
if (phase === "scanning_tables") return "Reading tables.";
|
||||
if (phase === "scanning_columns") return "Reading columns and primary keys.";
|
||||
return "Reading foreign-key relationships.";
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
import { readFile } from "node:fs/promises";
|
||||
import type { WorkspaceSecretStore } from "../workspaces/secret-store.js";
|
||||
import type { CatalogPostgresAccess } from "./postgres-access.js";
|
||||
import { CATALOG_SECRET_IDS } from "./secrets.js";
|
||||
import {
|
||||
CatalogConnectorError,
|
||||
type ObservedCatalogTable,
|
||||
type WorkspaceDatabase,
|
||||
} from "./types.js";
|
||||
|
||||
export interface CatalogTableIntrospector {
|
||||
scan(database: WorkspaceDatabase, signal: AbortSignal): Promise<ObservedCatalogTable[]>;
|
||||
}
|
||||
|
||||
function normalize(rows: readonly ObservedCatalogTable[]): ObservedCatalogTable[] {
|
||||
const byName = new Map<string, ObservedCatalogTable>();
|
||||
for (const row of rows) {
|
||||
if (typeof row.name !== "string" || row.name.length === 0 || row.name.length > 128) {
|
||||
throw new CatalogConnectorError("Schema introspection response is invalid");
|
||||
}
|
||||
if (row.sourceComment !== null && typeof row.sourceComment !== "string") {
|
||||
throw new CatalogConnectorError("Schema introspection response is invalid");
|
||||
}
|
||||
byName.set(row.name, row);
|
||||
}
|
||||
return [...byName.values()]
|
||||
.sort((left, right) => left.name.localeCompare(right.name));
|
||||
}
|
||||
|
||||
function requireText(value: string | undefined): string {
|
||||
if (!value) throw new CatalogConnectorError("Database binding is incomplete");
|
||||
return value;
|
||||
}
|
||||
|
||||
export class ConcreteCatalogTableIntrospector implements CatalogTableIntrospector {
|
||||
constructor(
|
||||
private readonly postgres: CatalogPostgresAccess,
|
||||
private readonly secretStore: WorkspaceSecretStore,
|
||||
) {}
|
||||
|
||||
async scan(database: WorkspaceDatabase, signal: AbortSignal): Promise<ObservedCatalogTable[]> {
|
||||
return database.binding.transport === "rest_api"
|
||||
? await this.scanRest(database, signal)
|
||||
: await this.scanPostgres(database, signal);
|
||||
}
|
||||
|
||||
private async scanPostgres(
|
||||
database: WorkspaceDatabase,
|
||||
signal: AbortSignal,
|
||||
): Promise<ObservedCatalogTable[]> {
|
||||
const client = await this.postgres.connect(database, signal);
|
||||
try {
|
||||
const schema = await client.query(
|
||||
"SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname = $1) AS present",
|
||||
[database.schema],
|
||||
);
|
||||
if (schema.rows[0]?.present !== true) throw new CatalogConnectorError("Database schema is unavailable");
|
||||
const result = await client.query(
|
||||
`SELECT c.relname AS name, d.description AS source_comment
|
||||
FROM pg_catalog.pg_class c
|
||||
JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
|
||||
LEFT JOIN pg_catalog.pg_description d ON d.objoid = c.oid AND d.objsubid = 0
|
||||
WHERE c.relkind IN ('r', 'p') AND n.nspname = $1
|
||||
ORDER BY c.relname`,
|
||||
[database.schema],
|
||||
);
|
||||
return normalize(result.rows.map((row) => ({
|
||||
name: String(row.name),
|
||||
sourceComment: typeof row.source_comment === "string" && row.source_comment.length > 0
|
||||
? row.source_comment
|
||||
: null,
|
||||
})));
|
||||
} finally {
|
||||
await client.end();
|
||||
}
|
||||
}
|
||||
|
||||
private async scanRest(
|
||||
database: WorkspaceDatabase,
|
||||
signal: AbortSignal,
|
||||
): Promise<ObservedCatalogTable[]> {
|
||||
const auth = database.binding.restAuth ?? "bearer";
|
||||
const required = auth === "none" ? [] : [CATALOG_SECRET_IDS.apiKey];
|
||||
const materialized = this.secretStore.materialize(database.workspaceId, required);
|
||||
try {
|
||||
const headers: Record<string, string> = { "content-type": "application/json" };
|
||||
if (auth !== "none") {
|
||||
const credentialFile = materialized.files.get(CATALOG_SECRET_IDS.apiKey);
|
||||
if (!credentialFile) throw new CatalogConnectorError("REST API key is not configured");
|
||||
const credential = (await readFile(credentialFile, "utf8")).trim();
|
||||
if (auth === "bearer") headers.authorization = `Bearer ${credential}`;
|
||||
else headers["x-api-key"] = credential;
|
||||
}
|
||||
const baseUrl = requireText(database.binding.baseUrl).replace(/\/+$/, "");
|
||||
const response = await fetch(`${baseUrl}/rpc/list_tables`, {
|
||||
method: "POST",
|
||||
headers,
|
||||
body: JSON.stringify({ schema_name: database.schema }),
|
||||
signal,
|
||||
});
|
||||
if (!response.ok) throw new CatalogConnectorError("REST schema introspection failed");
|
||||
const body: unknown = await response.json();
|
||||
if (!Array.isArray(body)) throw new CatalogConnectorError("REST schema response is invalid");
|
||||
const rows: ObservedCatalogTable[] = [];
|
||||
for (const item of body) {
|
||||
if (!item || typeof item !== "object") {
|
||||
throw new CatalogConnectorError("REST schema response is invalid");
|
||||
}
|
||||
const row = item as Record<string, unknown>;
|
||||
if (typeof row.type !== "string") {
|
||||
throw new CatalogConnectorError("REST schema response is invalid");
|
||||
}
|
||||
if (row.type !== "TABLE") continue;
|
||||
if (typeof row.table !== "string" || row.table.length === 0 || row.table.length > 128) {
|
||||
throw new CatalogConnectorError("REST schema response is invalid");
|
||||
}
|
||||
if (row.comment !== undefined && row.comment !== null && typeof row.comment !== "string") {
|
||||
throw new CatalogConnectorError("REST schema response is invalid");
|
||||
}
|
||||
rows.push({
|
||||
name: row.table,
|
||||
sourceComment: typeof row.comment === "string" && row.comment.length > 0 ? row.comment : null,
|
||||
});
|
||||
}
|
||||
return normalize(rows);
|
||||
} catch (error) {
|
||||
if (error instanceof CatalogConnectorError) throw error;
|
||||
throw new CatalogConnectorError("REST schema introspection failed");
|
||||
} finally {
|
||||
materialized.release();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
import type {
|
||||
CatalogRepository,
|
||||
CatalogTable,
|
||||
} from "./types.js";
|
||||
|
||||
export class CatalogTableService {
|
||||
constructor(
|
||||
private readonly repository: CatalogRepository,
|
||||
) {}
|
||||
|
||||
async list(databaseId: string): Promise<CatalogTable[]> {
|
||||
return await this.repository.listTables(databaseId);
|
||||
}
|
||||
|
||||
async updateDescription(
|
||||
databaseId: string,
|
||||
tableId: string,
|
||||
expectedVersion: number,
|
||||
description: string | null,
|
||||
): Promise<CatalogTable | undefined> {
|
||||
const normalized = description?.trim() || null;
|
||||
return await this.repository.updateTableDescription(
|
||||
databaseId,
|
||||
tableId,
|
||||
expectedVersion,
|
||||
normalized,
|
||||
);
|
||||
}
|
||||
|
||||
async updateMetadata(
|
||||
databaseId: string,
|
||||
tableId: string,
|
||||
expectedVersion: number,
|
||||
description: string | null,
|
||||
generatedDescription: string | null,
|
||||
): Promise<CatalogTable | undefined> {
|
||||
return await this.repository.updateTableMetadata(
|
||||
databaseId,
|
||||
tableId,
|
||||
expectedVersion,
|
||||
description?.trim() || null,
|
||||
generatedDescription?.trim() || null,
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,339 @@
|
||||
export const DATABASE_TRANSPORTS = ["postgres_direct", "rest_api", "ssh_tunnel"] as const;
|
||||
export type DatabaseTransport = (typeof DATABASE_TRANSPORTS)[number];
|
||||
|
||||
export type ConnectionStatus = "untested" | "reachable" | "failed";
|
||||
|
||||
export interface DatabaseBinding {
|
||||
transport: DatabaseTransport;
|
||||
host?: string;
|
||||
port?: number;
|
||||
username?: string;
|
||||
baseUrl?: string;
|
||||
restPath?: string;
|
||||
restAuth?: "none" | "bearer" | "x-api-key";
|
||||
tlsServername?: string;
|
||||
sshHost?: string;
|
||||
sshPort?: number;
|
||||
sshUsername?: string;
|
||||
sshTargetHost?: string;
|
||||
sshTargetPort?: number;
|
||||
}
|
||||
|
||||
export interface WorkspaceDatabase {
|
||||
id: string;
|
||||
workspaceId: string;
|
||||
engine: "postgres";
|
||||
databaseName: string;
|
||||
schema: string;
|
||||
version: number;
|
||||
createdAt: string;
|
||||
updatedAt: string;
|
||||
binding: DatabaseBinding;
|
||||
connectionStatus: ConnectionStatus;
|
||||
testedVersion?: number;
|
||||
lastTestedAt?: string;
|
||||
lastErrorCode?: string;
|
||||
lastErrorMessage?: string;
|
||||
schemaSyncedVersion?: number;
|
||||
schemaSyncedAt?: string;
|
||||
}
|
||||
|
||||
export interface DatabaseConfigurationInput {
|
||||
workspaceId: string;
|
||||
engine: "postgres";
|
||||
databaseName: string;
|
||||
schema: string;
|
||||
binding: DatabaseBinding;
|
||||
}
|
||||
|
||||
export interface DatabaseTestResult {
|
||||
connectionStatus: Exclude<ConnectionStatus, "untested">;
|
||||
testedVersion: number;
|
||||
lastTestedAt: string;
|
||||
errorCode?: string;
|
||||
errorMessage?: string;
|
||||
}
|
||||
|
||||
export interface CatalogTable {
|
||||
id: string;
|
||||
databaseId: string;
|
||||
name: string;
|
||||
sourceComment: string | null;
|
||||
description: string | null;
|
||||
generatedDescription: string | null;
|
||||
lastSyncedDatabaseVersion: number | null;
|
||||
lastSyncedAt: string | null;
|
||||
version: number;
|
||||
createdAt: string;
|
||||
updatedAt: string;
|
||||
}
|
||||
|
||||
export interface ObservedCatalogTable {
|
||||
name: string;
|
||||
sourceComment: string | null;
|
||||
}
|
||||
|
||||
export interface CatalogColumn {
|
||||
id: string;
|
||||
tableId: string;
|
||||
name: string;
|
||||
ordinalPosition: number;
|
||||
dataType: string;
|
||||
isNullable: boolean;
|
||||
defaultExpression: string | null;
|
||||
primaryKeyPosition: number | null;
|
||||
isPrimaryKey: boolean;
|
||||
isForeignKey: boolean;
|
||||
foreignKeyCount: number;
|
||||
sourceComment: string | null;
|
||||
description: string | null;
|
||||
generatedDescription: string | null;
|
||||
lastSyncedDatabaseVersion: number | null;
|
||||
lastSyncedAt: string | null;
|
||||
version: number;
|
||||
createdAt: string;
|
||||
updatedAt: string;
|
||||
}
|
||||
|
||||
export interface ObservedCatalogColumn {
|
||||
tableName: string;
|
||||
name: string;
|
||||
ordinalPosition: number;
|
||||
dataType: string;
|
||||
isNullable: boolean;
|
||||
defaultExpression: string | null;
|
||||
primaryKeyPosition: number | null;
|
||||
sourceComment: string | null;
|
||||
}
|
||||
|
||||
export interface CatalogRelationshipColumn {
|
||||
position: number;
|
||||
sourceColumnId: string;
|
||||
sourceColumnName: string;
|
||||
targetColumnId: string;
|
||||
targetColumnName: string;
|
||||
}
|
||||
|
||||
export interface CatalogRelationship {
|
||||
id: string;
|
||||
databaseId: string;
|
||||
constraintName: string;
|
||||
sourceTableId: string;
|
||||
sourceTableName: string;
|
||||
targetTableId: string;
|
||||
targetTableName: string;
|
||||
updateRule: string;
|
||||
deleteRule: string;
|
||||
deferrable: boolean;
|
||||
initiallyDeferred: boolean;
|
||||
columns: CatalogRelationshipColumn[];
|
||||
lastSyncedDatabaseVersion: number | null;
|
||||
lastSyncedAt: string | null;
|
||||
createdAt: string;
|
||||
updatedAt: string;
|
||||
}
|
||||
|
||||
export interface ObservedRelationshipColumn {
|
||||
position: number;
|
||||
sourceColumnName: string;
|
||||
targetColumnName: string;
|
||||
}
|
||||
|
||||
export interface ObservedCatalogRelationship {
|
||||
constraintName: string;
|
||||
sourceTableName: string;
|
||||
targetTableName: string;
|
||||
updateRule: string;
|
||||
deleteRule: string;
|
||||
deferrable: boolean;
|
||||
initiallyDeferred: boolean;
|
||||
columns: ObservedRelationshipColumn[];
|
||||
}
|
||||
|
||||
export type IntrospectionCapabilityState = "available" | "unavailable";
|
||||
export interface ObservedSchemaSnapshot {
|
||||
schemaVersion: 1;
|
||||
capabilities: {
|
||||
tables: IntrospectionCapabilityState;
|
||||
columns: IntrospectionCapabilityState;
|
||||
relationships: IntrospectionCapabilityState;
|
||||
};
|
||||
tables: ObservedCatalogTable[];
|
||||
columns: ObservedCatalogColumn[];
|
||||
relationships: ObservedCatalogRelationship[];
|
||||
}
|
||||
|
||||
export type CatalogSyncScope = "tables" | "columns" | "relationships" | "all";
|
||||
export type CatalogSyncState =
|
||||
| "queued" | "running" | "awaiting_confirmation" | "applying"
|
||||
| "succeeded" | "failed" | "cancelled" | "interrupted";
|
||||
export type CatalogSyncPhase =
|
||||
| "queued" | "connecting" | "scanning_tables" | "scanning_columns"
|
||||
| "scanning_relationships" | "planning" | "awaiting_confirmation"
|
||||
| "applying" | "completed";
|
||||
|
||||
export interface CatalogSchemaDiff {
|
||||
deletedTables: string[];
|
||||
deletedColumns: Array<{ tableName: string; columnName: string }>;
|
||||
deletedRelationships: Array<{ sourceTableName: string; constraintName: string }>;
|
||||
}
|
||||
|
||||
export interface CatalogSyncCounts {
|
||||
tables?: number;
|
||||
columns?: number;
|
||||
relationships?: number;
|
||||
created?: number;
|
||||
updated?: number;
|
||||
deleted?: number;
|
||||
}
|
||||
|
||||
export interface CatalogSyncRun {
|
||||
id: string;
|
||||
databaseId: string;
|
||||
scope: CatalogSyncScope;
|
||||
tableIds: string[];
|
||||
state: CatalogSyncState;
|
||||
phase: CatalogSyncPhase;
|
||||
requestedDatabaseVersion: number;
|
||||
observedSnapshot: ObservedSchemaSnapshot | null;
|
||||
plannedDiff: CatalogSchemaDiff | null;
|
||||
confirmationToken: string | null;
|
||||
counts: CatalogSyncCounts;
|
||||
errorCode: string | null;
|
||||
errorMessage: string | null;
|
||||
cancelRequested: boolean;
|
||||
createdAt: string;
|
||||
startedAt: string | null;
|
||||
updatedAt: string;
|
||||
finishedAt: string | null;
|
||||
heartbeatAt: string | null;
|
||||
leaseOwner: string | null;
|
||||
leaseExpiresAt: string | null;
|
||||
}
|
||||
|
||||
export interface CatalogSyncEvent {
|
||||
id: number;
|
||||
runId: string;
|
||||
sequence: number;
|
||||
level: "info" | "warning" | "error";
|
||||
eventType: string;
|
||||
message: string;
|
||||
data: Record<string, unknown>;
|
||||
createdAt: string;
|
||||
}
|
||||
|
||||
export interface CatalogSyncRunUpdate {
|
||||
state?: CatalogSyncState;
|
||||
phase?: CatalogSyncPhase;
|
||||
observedSnapshot?: ObservedSchemaSnapshot | null;
|
||||
plannedDiff?: CatalogSchemaDiff | null;
|
||||
confirmationToken?: string | null;
|
||||
counts?: CatalogSyncCounts;
|
||||
errorCode?: string | null;
|
||||
errorMessage?: string | null;
|
||||
cancelRequested?: boolean;
|
||||
startedAt?: string | null;
|
||||
finishedAt?: string | null;
|
||||
heartbeatAt?: string | null;
|
||||
leaseOwner?: string | null;
|
||||
leaseExpiresAt?: string | null;
|
||||
}
|
||||
|
||||
export type TableSyncRepositoryResult =
|
||||
| { kind: "confirmation_required"; deletedNames: string[] }
|
||||
| {
|
||||
kind: "applied";
|
||||
createdCount: number;
|
||||
updatedCount: number;
|
||||
deletedCount: number;
|
||||
tables: CatalogTable[];
|
||||
};
|
||||
|
||||
export interface CatalogRepository {
|
||||
list(): Promise<WorkspaceDatabase[]>;
|
||||
get(id: string): Promise<WorkspaceDatabase | undefined>;
|
||||
getByWorkspace(workspaceId: string): Promise<WorkspaceDatabase | undefined>;
|
||||
create(input: DatabaseConfigurationInput): Promise<WorkspaceDatabase>;
|
||||
update(id: string, expectedVersion: number, input: DatabaseConfigurationInput): Promise<WorkspaceDatabase | undefined>;
|
||||
recordTest(id: string, expectedVersion: number, result: DatabaseTestResult): Promise<WorkspaceDatabase | undefined>;
|
||||
touch(id: string, expectedVersion: number): Promise<WorkspaceDatabase | undefined>;
|
||||
delete(id: string, expectedVersion: number): Promise<boolean>;
|
||||
listTables(databaseId: string): Promise<CatalogTable[]>;
|
||||
getTable(databaseId: string, tableId: string): Promise<CatalogTable | undefined>;
|
||||
updateTableDescription(
|
||||
databaseId: string,
|
||||
tableId: string,
|
||||
expectedVersion: number,
|
||||
description: string | null,
|
||||
): Promise<CatalogTable | undefined>;
|
||||
updateTableMetadata(
|
||||
databaseId: string,
|
||||
tableId: string,
|
||||
expectedVersion: number,
|
||||
description: string | null,
|
||||
generatedDescription: string | null,
|
||||
): Promise<CatalogTable | undefined>;
|
||||
listColumns(databaseId: string, tableId: string): Promise<CatalogColumn[]>;
|
||||
getColumn(databaseId: string, tableId: string, columnId: string): Promise<CatalogColumn | undefined>;
|
||||
updateColumnMetadata(
|
||||
databaseId: string,
|
||||
tableId: string,
|
||||
columnId: string,
|
||||
expectedVersion: number,
|
||||
description: string | null,
|
||||
generatedDescription: string | null,
|
||||
): Promise<CatalogColumn | undefined>;
|
||||
listRelationships(databaseId: string): Promise<CatalogRelationship[]>;
|
||||
planSchemaSync(
|
||||
databaseId: string,
|
||||
scope: CatalogSyncScope,
|
||||
tableIds: readonly string[],
|
||||
snapshot: ObservedSchemaSnapshot,
|
||||
): Promise<CatalogSchemaDiff>;
|
||||
applySchemaSync(
|
||||
databaseId: string,
|
||||
expectedDatabaseVersion: number,
|
||||
scope: CatalogSyncScope,
|
||||
tableIds: readonly string[],
|
||||
snapshot: ObservedSchemaSnapshot,
|
||||
): Promise<CatalogSyncCounts | undefined>;
|
||||
createSyncRun(
|
||||
databaseId: string,
|
||||
scope: CatalogSyncScope,
|
||||
tableIds: readonly string[],
|
||||
requestedDatabaseVersion: number,
|
||||
): Promise<CatalogSyncRun>;
|
||||
getSyncRun(runId: string): Promise<CatalogSyncRun | undefined>;
|
||||
claimSyncRun(runId: string, workerId: string, leaseExpiresAt: string): Promise<CatalogSyncRun | undefined>;
|
||||
listSyncRuns(databaseId: string, limit?: number): Promise<CatalogSyncRun[]>;
|
||||
updateSyncRun(runId: string, update: CatalogSyncRunUpdate): Promise<CatalogSyncRun | undefined>;
|
||||
requestSyncRunCancellation(runId: string): Promise<CatalogSyncRun | undefined>;
|
||||
appendSyncEvent(
|
||||
runId: string,
|
||||
level: CatalogSyncEvent["level"],
|
||||
eventType: string,
|
||||
message: string,
|
||||
data?: Record<string, unknown>,
|
||||
): Promise<CatalogSyncEvent>;
|
||||
listSyncEvents(runId: string, afterSequence?: number): Promise<CatalogSyncEvent[]>;
|
||||
pruneSyncEvents(before: string): Promise<void>;
|
||||
interruptActiveSyncRuns(): Promise<void>;
|
||||
reconcileTables(
|
||||
databaseId: string,
|
||||
expectedDatabaseVersion: number,
|
||||
observed: readonly ObservedCatalogTable[],
|
||||
confirmedDeletedNames: readonly string[],
|
||||
): Promise<TableSyncRepositoryResult | undefined>;
|
||||
available(): Promise<boolean>;
|
||||
close?(): Promise<void>;
|
||||
}
|
||||
|
||||
export class CatalogConflictError extends Error {}
|
||||
export class CatalogUnavailableError extends Error {}
|
||||
export class CatalogOperationInProgressError extends Error {}
|
||||
export class CatalogConnectorError extends Error {}
|
||||
export class CatalogSchemaCapabilityUnavailableError extends CatalogConnectorError {
|
||||
constructor(readonly capability: "schema_snapshot" | "tables" | "columns" | "relationships") {
|
||||
super(`Schema introspection capability '${capability}' is unavailable`);
|
||||
}
|
||||
}
|
||||
@@ -7,6 +7,7 @@ import {
|
||||
} from "./auth/config.js";
|
||||
import { createProjectedAuthenticationConfigProvider } from "./auth/runtime-projection.js";
|
||||
import type { WorkspaceRegistryConfig } from "./workspaces/types.js";
|
||||
import type { CatalogConnectionConfig } from "./catalog/repository.js";
|
||||
|
||||
export interface AppConfig {
|
||||
host: string; port: number; harnessDir: string; thtBin: string; piBin: string;
|
||||
@@ -20,6 +21,8 @@ export interface AppConfig {
|
||||
host?: string; port?: number; database?: string; runtimeUser?: string;
|
||||
runtimePasswordFile?: string; sslmode?: "verify-ca" | "verify-full"; sslrootcert?: string;
|
||||
};
|
||||
/** Installation-local metadata catalog. Omitted installations expose an unavailable admin surface. */
|
||||
catalogDatabase?: CatalogConnectionConfig;
|
||||
defaults: { provider?: string; model?: string; thinking?: string };
|
||||
maxPiProcesses: number;
|
||||
settingsFile: string;
|
||||
@@ -40,6 +43,7 @@ export interface AppConfig {
|
||||
/** Explicit compatibility mode for old loopback clients that send `workspace` in POST /sessions. */
|
||||
legacyWorkspaceMode: boolean;
|
||||
workspaceDiagnosticTimeoutMs: number;
|
||||
catalogSyncTimeoutMs: number;
|
||||
workspaceRegistry: WorkspaceRegistryConfig;
|
||||
workspaceSecretStoreRoot: string;
|
||||
workspaceSecretRuntimeRoot: string;
|
||||
@@ -127,6 +131,14 @@ function diagnosticTimeout(value: string | undefined): number {
|
||||
return timeout;
|
||||
}
|
||||
|
||||
function catalogSyncTimeout(value: string | undefined): number {
|
||||
const timeout = Number(value ?? 600_000);
|
||||
if (!Number.isSafeInteger(timeout) || timeout < 1_000 || timeout > 3_600_000) {
|
||||
throw new Error("catalog synchronization timeout configuration is invalid");
|
||||
}
|
||||
return timeout;
|
||||
}
|
||||
|
||||
function piManagementTimeout(value: string | undefined): number {
|
||||
const timeout = Number(value ?? 8_000);
|
||||
if (!Number.isSafeInteger(timeout) || timeout < 1 || timeout > 30_000) {
|
||||
@@ -176,6 +188,33 @@ function positiveDimension(value: string | undefined, fallback: number): number
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function catalogDatabase(env: Record<string, string | undefined>): CatalogConnectionConfig | undefined {
|
||||
const value = env.THT_CATALOG_DATABASE_URL;
|
||||
if (value !== undefined) {
|
||||
try {
|
||||
const parsed = new URL(value);
|
||||
if ((parsed.protocol !== "postgres:" && parsed.protocol !== "postgresql:")
|
||||
|| !parsed.hostname || !parsed.pathname.slice(1) || parsed.hash || parsed.search) throw new Error();
|
||||
return { connectionString: value };
|
||||
} catch {
|
||||
throw new Error("catalog database configuration is invalid");
|
||||
}
|
||||
}
|
||||
const host = env.THT_CATALOG_DB_HOST;
|
||||
if (host === undefined) return undefined;
|
||||
const port = Number(env.THT_CATALOG_DB_PORT ?? 5432);
|
||||
const database = env.THT_CATALOG_DB_NAME;
|
||||
const user = env.THT_CATALOG_RUNTIME_USER;
|
||||
const passwordFile = env.THT_CATALOG_RUNTIME_PASSWORD_FILE;
|
||||
try {
|
||||
if (!host.trim() || !database?.trim() || !user?.trim() || !passwordFile
|
||||
|| !path.isAbsolute(passwordFile) || !Number.isInteger(port) || port < 1 || port > 65_535) throw new Error();
|
||||
return { host, port, database, user, passwordFile };
|
||||
} catch {
|
||||
throw new Error("catalog database configuration is invalid");
|
||||
}
|
||||
}
|
||||
|
||||
export function loadConfig(
|
||||
env: Record<string, string | undefined>,
|
||||
options: { surface?: "application" | "workspace-maintenance" } = {},
|
||||
@@ -356,6 +395,7 @@ export function loadConfig(
|
||||
authentication,
|
||||
publicExposure,
|
||||
sessionStorage,
|
||||
catalogDatabase: catalogDatabase(env),
|
||||
defaults: { provider: env.PI_PROVIDER, model: env.PI_MODEL, thinking: env.PI_THINKING },
|
||||
maxPiProcesses: Number(env.MAX_PI_PROCESSES ?? 4),
|
||||
settingsFile,
|
||||
@@ -370,6 +410,7 @@ export function loadConfig(
|
||||
dwhPrecheck: env.THT_DWH_PRECHECK === "true" || env.THT_DWH_PRECHECK === "1",
|
||||
legacyWorkspaceMode: legacyWorkspaceMode === "local",
|
||||
workspaceDiagnosticTimeoutMs: diagnosticTimeout(env.THT_WORKSPACE_DIAGNOSTIC_TIMEOUT_MS),
|
||||
catalogSyncTimeoutMs: catalogSyncTimeout(env.THT_CATALOG_SYNC_TIMEOUT_MS),
|
||||
workspaceRegistry,
|
||||
workspaceSecretStoreRoot,
|
||||
workspaceSecretRuntimeRoot,
|
||||
|
||||
@@ -0,0 +1,218 @@
|
||||
import type { FastifyInstance, FastifyReply, FastifyRequest } from "fastify";
|
||||
import { z } from "zod";
|
||||
import { isPrincipalContext, requirePermission } from "../auth/authorization.js";
|
||||
import { CatalogService, type CatalogSecretName } from "../catalog/service.js";
|
||||
import { WorkspaceRegistryError } from "../workspaces/git-repository.js";
|
||||
import {
|
||||
CatalogConflictError,
|
||||
CatalogOperationInProgressError,
|
||||
CatalogUnavailableError,
|
||||
DATABASE_TRANSPORTS,
|
||||
type CatalogRepository,
|
||||
type DatabaseConfigurationInput,
|
||||
} from "../catalog/types.js";
|
||||
import type { CatalogOperationCoordinator } from "../catalog/operation-coordinator.js";
|
||||
|
||||
const idSchema = z.uuid();
|
||||
const workspaceIdSchema = z.string().regex(/^[a-z][a-z0-9-]{2,62}$/);
|
||||
const identifier = z.string().trim().min(1).max(128).regex(/^[A-Za-z_][A-Za-z0-9_$-]*$/);
|
||||
const nonEmpty = z.string().trim().min(1).max(512);
|
||||
const port = z.number().int().min(1).max(65_535);
|
||||
const optionalText = nonEmpty.optional();
|
||||
const sshHost = z.string().trim().min(1).max(255).regex(/^[A-Za-z0-9_.:\[\]-]+$/).optional();
|
||||
const sshUsername = z.string().trim().min(1).max(128).regex(/^[A-Za-z0-9._-]+$/).optional();
|
||||
const bindingSchema = z.object({
|
||||
transport: z.enum(DATABASE_TRANSPORTS),
|
||||
host: optionalText,
|
||||
port: port.optional(),
|
||||
username: optionalText,
|
||||
baseUrl: z.url().max(2048).optional(),
|
||||
restPath: z.string().regex(/^\/(?!\/)[^?#\\\u0000-\u001f]*$/).max(512).optional(),
|
||||
restAuth: z.enum(["none", "bearer", "x-api-key"]).optional(),
|
||||
tlsServername: optionalText,
|
||||
sshHost,
|
||||
sshPort: port.optional(),
|
||||
sshUsername,
|
||||
sshTargetHost: sshHost,
|
||||
sshTargetPort: port.optional(),
|
||||
}).strict().superRefine((binding, context) => {
|
||||
const required = binding.transport === "postgres_direct"
|
||||
? ["host", "port", "username"] as const
|
||||
: binding.transport === "rest_api"
|
||||
? ["baseUrl", "restPath", "restAuth"] as const
|
||||
: ["username", "sshHost", "sshPort", "sshUsername", "sshTargetHost", "sshTargetPort"] as const;
|
||||
for (const field of required) {
|
||||
if (binding[field] === undefined) context.addIssue({ code: "custom", path: [field], message: "Required" });
|
||||
}
|
||||
});
|
||||
const configSchema = z.object({
|
||||
workspaceId: workspaceIdSchema,
|
||||
engine: z.literal("postgres"),
|
||||
databaseName: identifier,
|
||||
schema: identifier,
|
||||
binding: bindingSchema,
|
||||
}).strict();
|
||||
const updateSchema = configSchema.extend({ version: z.number().int().positive() });
|
||||
const secretNames = [
|
||||
"password",
|
||||
"apiKey",
|
||||
"sshPrivateKey",
|
||||
"sshPrivateKeyPassphrase",
|
||||
"sshKnownHosts",
|
||||
"tlsCa",
|
||||
] as const;
|
||||
const secretsSchema = z.object({
|
||||
version: z.number().int().positive(),
|
||||
values: z.partialRecord(z.enum(secretNames), z.string().min(1).max(65_536)).refine((values) => Object.keys(values).length > 0),
|
||||
}).strict();
|
||||
const versionQuery = z.object({ version: z.coerce.number().int().positive() });
|
||||
|
||||
function safeError(reply: FastifyReply, error: unknown) {
|
||||
if (error instanceof CatalogUnavailableError) {
|
||||
return reply.code(503).send({ code: "catalog_unavailable", message: "Database catalog is unavailable." });
|
||||
}
|
||||
if (error instanceof CatalogConflictError) {
|
||||
return reply.code(409).send({ code: "database_conflict", message: "This workspace already has a database configuration." });
|
||||
}
|
||||
if (error instanceof CatalogOperationInProgressError) {
|
||||
return reply.code(409).send({ code: "database_operation_in_progress", message: "A database operation is already in progress." });
|
||||
}
|
||||
if (error instanceof z.ZodError) {
|
||||
return reply.code(400).send({ code: "database_invalid", message: "Database configuration is invalid." });
|
||||
}
|
||||
if (error instanceof WorkspaceRegistryError) {
|
||||
return reply.code(400).send({ code: "database_invalid", message: "Database configuration is invalid." });
|
||||
}
|
||||
return reply.code(500).send({ code: "database_operation_failed", message: "Database operation failed." });
|
||||
}
|
||||
|
||||
function manage(request: FastifyRequest, reply: FastifyReply) {
|
||||
return isPrincipalContext(requirePermission(request, reply, "database.manage"));
|
||||
}
|
||||
|
||||
export function catalogDatabaseRoutes(
|
||||
app: FastifyInstance,
|
||||
deps: { repository: CatalogRepository; service: CatalogService; operations?: CatalogOperationCoordinator },
|
||||
): void {
|
||||
const mutate = async <T>(databaseId: string, operation: () => Promise<T>): Promise<T> => (
|
||||
deps.operations ? await deps.operations.run(databaseId, operation) : await operation()
|
||||
);
|
||||
const activeSyncRun = async (databaseId: string) => {
|
||||
const run = (await deps.repository.listSyncRuns(databaseId, 5)).find((candidate) =>
|
||||
["queued", "running", "awaiting_confirmation", "applying"].includes(candidate.state));
|
||||
if (!run) return undefined;
|
||||
const {
|
||||
observedSnapshot: _snapshot,
|
||||
leaseOwner: _leaseOwner,
|
||||
leaseExpiresAt: _leaseExpiresAt,
|
||||
...summary
|
||||
} = run;
|
||||
return summary;
|
||||
};
|
||||
app.get("/catalog/status", async (request, reply) => {
|
||||
if (!manage(request, reply)) return reply;
|
||||
return { available: await deps.repository.available() };
|
||||
});
|
||||
|
||||
app.get("/catalog/databases", async (request, reply) => {
|
||||
if (!manage(request, reply)) return reply;
|
||||
try {
|
||||
const rows = await deps.service.list();
|
||||
return await Promise.all(rows.map(async (row) => (
|
||||
row.id ? { ...row, activeSyncRun: await activeSyncRun(row.id) } : row
|
||||
)));
|
||||
} catch (error) { return safeError(reply, error); }
|
||||
});
|
||||
|
||||
app.get("/catalog/databases/:id", async (request, reply) => {
|
||||
if (!manage(request, reply)) return reply;
|
||||
try {
|
||||
const id = idSchema.parse((request.params as { id?: unknown }).id);
|
||||
const database = await deps.repository.get(id);
|
||||
if (!database) return reply.code(404).send({ code: "database_not_found", message: "Database configuration was not found." });
|
||||
return {
|
||||
...database,
|
||||
configured: true,
|
||||
secrets: deps.service.configuredSecrets(database.workspaceId),
|
||||
activeSyncRun: await activeSyncRun(database.id),
|
||||
};
|
||||
} catch (error) { return safeError(reply, error); }
|
||||
});
|
||||
|
||||
app.post("/catalog/databases", async (request, reply) => {
|
||||
if (!manage(request, reply)) return reply;
|
||||
try {
|
||||
const input = configSchema.parse(request.body) as DatabaseConfigurationInput;
|
||||
const created = await deps.repository.create(await deps.service.normalizeInput(input));
|
||||
return reply.code(201).send({ ...created, configured: true, secrets: deps.service.configuredSecrets(created.workspaceId) });
|
||||
} catch (error) { return safeError(reply, error); }
|
||||
});
|
||||
|
||||
app.patch("/catalog/databases/:id", async (request, reply) => {
|
||||
if (!manage(request, reply)) return reply;
|
||||
try {
|
||||
const id = idSchema.parse((request.params as { id?: unknown }).id);
|
||||
const { version, ...input } = updateSchema.parse(request.body);
|
||||
const current = await deps.repository.get(id);
|
||||
if (!current) return reply.code(404).send({ code: "database_not_found", message: "Database configuration was not found." });
|
||||
if (current.workspaceId !== input.workspaceId) {
|
||||
return reply.code(400).send({ code: "database_invalid", message: "Database configuration is invalid." });
|
||||
}
|
||||
const updated = await mutate(id, async () => await deps.repository.update(
|
||||
id, version, await deps.service.normalizeInput(input as DatabaseConfigurationInput),
|
||||
));
|
||||
if (!updated) return reply.code(409).send({ code: "database_stale", message: "Database configuration changed. Reload and try again." });
|
||||
return { ...updated, configured: true, secrets: deps.service.configuredSecrets(updated.workspaceId) };
|
||||
} catch (error) { return safeError(reply, error); }
|
||||
});
|
||||
|
||||
app.put("/catalog/databases/:id/secrets", async (request, reply) => {
|
||||
if (!manage(request, reply)) return reply;
|
||||
if (!isPrincipalContext(requirePermission(request, reply, "workspace.secrets.manage"))) return reply;
|
||||
try {
|
||||
const id = idSchema.parse((request.params as { id?: unknown }).id);
|
||||
const { version, values } = secretsSchema.parse(request.body);
|
||||
const database = await deps.repository.get(id);
|
||||
if (!database) return reply.code(404).send({ code: "database_not_found", message: "Database configuration was not found." });
|
||||
if (database.version !== version) return reply.code(409).send({ code: "database_stale", message: "Database configuration changed. Reload and try again." });
|
||||
const updated = await mutate(id, async () => {
|
||||
const touched = await deps.repository.touch(id, version);
|
||||
if (touched) deps.service.replaceSecrets(database.workspaceId, values as Partial<Record<CatalogSecretName, string>>);
|
||||
return touched;
|
||||
});
|
||||
if (!updated) return reply.code(409).send({ code: "database_stale", message: "Database configuration changed. Reload and try again." });
|
||||
return { ...updated, configured: true, secrets: deps.service.configuredSecrets(updated.workspaceId) };
|
||||
} catch (error) { return safeError(reply, error); }
|
||||
});
|
||||
|
||||
app.post("/catalog/databases/:id/test", async (request, reply) => {
|
||||
if (!manage(request, reply)) return reply;
|
||||
try {
|
||||
const id = idSchema.parse((request.params as { id?: unknown }).id);
|
||||
const { version } = z.object({ version: z.number().int().positive() }).strict().parse(request.body);
|
||||
const database = await deps.repository.get(id);
|
||||
if (!database) return reply.code(404).send({ code: "database_not_found", message: "Database configuration was not found." });
|
||||
if (database.version !== version) return reply.code(409).send({ code: "database_stale", message: "Database configuration changed. Reload and try again." });
|
||||
const tested = await deps.repository.recordTest(id, version, await deps.service.test(database));
|
||||
if (!tested) return reply.code(409).send({ code: "database_stale", message: "Database configuration changed. Reload and try again." });
|
||||
return { ...tested, configured: true, secrets: deps.service.configuredSecrets(tested.workspaceId) };
|
||||
} catch (error) { return safeError(reply, error); }
|
||||
});
|
||||
|
||||
app.delete("/catalog/databases/:id", async (request, reply) => {
|
||||
if (!manage(request, reply)) return reply;
|
||||
try {
|
||||
const id = idSchema.parse((request.params as { id?: unknown }).id);
|
||||
const { version } = versionQuery.parse(request.query);
|
||||
const database = await deps.repository.get(id);
|
||||
if (!database) return reply.code(404).send({ code: "database_not_found", message: "Database configuration was not found." });
|
||||
const deleted = await mutate(id, async () => {
|
||||
const removed = await deps.repository.delete(id, version);
|
||||
if (removed) deps.service.forgetSecrets(database.workspaceId);
|
||||
return removed;
|
||||
});
|
||||
if (!deleted) return reply.code(409).send({ code: "database_stale", message: "Database configuration changed. Reload and try again." });
|
||||
return reply.code(204).send();
|
||||
} catch (error) { return safeError(reply, error); }
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,230 @@
|
||||
import type { FastifyInstance, FastifyReply, FastifyRequest } from "fastify";
|
||||
import { setTimeout as delay } from "node:timers/promises";
|
||||
import { z } from "zod";
|
||||
import { isPrincipalContext, requirePermission } from "../auth/authorization.js";
|
||||
import type { CatalogSyncWorker } from "../catalog/sync-worker.js";
|
||||
import {
|
||||
CatalogConflictError,
|
||||
CatalogConnectorError,
|
||||
CatalogOperationInProgressError,
|
||||
CatalogUnavailableError,
|
||||
type CatalogRepository,
|
||||
type CatalogSyncRun,
|
||||
} from "../catalog/types.js";
|
||||
|
||||
const idSchema = z.uuid();
|
||||
const metadataSchema = z.object({
|
||||
version: z.number().int().positive(),
|
||||
description: z.string().max(20_000).nullable(),
|
||||
generatedDescription: z.string().max(20_000).nullable(),
|
||||
}).strict();
|
||||
const createRunSchema = z.object({
|
||||
version: z.number().int().positive(),
|
||||
scope: z.enum(["tables", "columns", "relationships", "all"]),
|
||||
tableIds: z.array(idSchema).max(10_000).default([]),
|
||||
}).strict();
|
||||
const confirmationSchema = z.object({ confirmationToken: z.string().uuid() }).strict();
|
||||
const eventQuerySchema = z.object({ after: z.coerce.number().int().nonnegative().default(0) });
|
||||
|
||||
function manage(request: FastifyRequest, reply: FastifyReply) {
|
||||
return isPrincipalContext(requirePermission(request, reply, "database.manage"));
|
||||
}
|
||||
|
||||
function safeError(reply: FastifyReply, error: unknown) {
|
||||
if (error instanceof CatalogUnavailableError) {
|
||||
return reply.code(503).send({ code: "catalog_unavailable", message: "Database catalog is unavailable." });
|
||||
}
|
||||
if (error instanceof CatalogConflictError || error instanceof CatalogOperationInProgressError) {
|
||||
return reply.code(409).send({ code: "schema_sync_conflict", message: error.message });
|
||||
}
|
||||
if (error instanceof CatalogConnectorError) {
|
||||
return reply.code(502).send({ code: "schema_introspection_failed", message: "The database schema could not be read safely." });
|
||||
}
|
||||
if (error instanceof z.ZodError) {
|
||||
return reply.code(400).send({ code: "schema_request_invalid", message: "Schema request is invalid." });
|
||||
}
|
||||
return reply.code(500).send({ code: "schema_operation_failed", message: "Schema operation failed." });
|
||||
}
|
||||
|
||||
function normalized(value: string | null): string | null {
|
||||
return value?.trim() || null;
|
||||
}
|
||||
|
||||
function publicRun(run: CatalogSyncRun) {
|
||||
const {
|
||||
observedSnapshot: _snapshot,
|
||||
leaseOwner: _leaseOwner,
|
||||
leaseExpiresAt: _leaseExpiresAt,
|
||||
...result
|
||||
} = run;
|
||||
return result;
|
||||
}
|
||||
|
||||
export function catalogSchemaRoutes(
|
||||
app: FastifyInstance,
|
||||
deps: { repository: CatalogRepository; worker: CatalogSyncWorker },
|
||||
): void {
|
||||
app.get("/catalog/databases/:databaseId/tables/:tableId/columns", async (request, reply) => {
|
||||
if (!manage(request, reply)) return reply;
|
||||
try {
|
||||
const params = request.params as { databaseId?: unknown; tableId?: unknown };
|
||||
const databaseId = idSchema.parse(params.databaseId);
|
||||
const tableId = idSchema.parse(params.tableId);
|
||||
if (!(await deps.repository.getTable(databaseId, tableId))) {
|
||||
return reply.code(404).send({ code: "table_not_found", message: "Catalog table was not found." });
|
||||
}
|
||||
return await deps.repository.listColumns(databaseId, tableId);
|
||||
} catch (error) { return safeError(reply, error); }
|
||||
});
|
||||
|
||||
app.patch("/catalog/databases/:databaseId/tables/:tableId/columns/:columnId", async (request, reply) => {
|
||||
if (!manage(request, reply)) return reply;
|
||||
try {
|
||||
const params = request.params as { databaseId?: unknown; tableId?: unknown; columnId?: unknown };
|
||||
const databaseId = idSchema.parse(params.databaseId);
|
||||
const tableId = idSchema.parse(params.tableId);
|
||||
const columnId = idSchema.parse(params.columnId);
|
||||
const input = metadataSchema.parse(request.body);
|
||||
const current = await deps.repository.getColumn(databaseId, tableId, columnId);
|
||||
if (!current) return reply.code(404).send({ code: "column_not_found", message: "Catalog column was not found." });
|
||||
if (current.version !== input.version) {
|
||||
return reply.code(409).send({ code: "column_stale", message: "Column metadata changed. Reload and try again." });
|
||||
}
|
||||
const updated = await deps.repository.updateColumnMetadata(
|
||||
databaseId,
|
||||
tableId,
|
||||
columnId,
|
||||
input.version,
|
||||
normalized(input.description),
|
||||
normalized(input.generatedDescription),
|
||||
);
|
||||
if (!updated) return reply.code(409).send({ code: "column_stale", message: "Column metadata changed. Reload and try again." });
|
||||
return updated;
|
||||
} catch (error) { return safeError(reply, error); }
|
||||
});
|
||||
|
||||
app.get("/catalog/databases/:databaseId/relationships", async (request, reply) => {
|
||||
if (!manage(request, reply)) return reply;
|
||||
try {
|
||||
const databaseId = idSchema.parse((request.params as { databaseId?: unknown }).databaseId);
|
||||
if (!(await deps.repository.get(databaseId))) {
|
||||
return reply.code(404).send({ code: "database_not_found", message: "Database configuration was not found." });
|
||||
}
|
||||
return await deps.repository.listRelationships(databaseId);
|
||||
} catch (error) { return safeError(reply, error); }
|
||||
});
|
||||
|
||||
app.post("/catalog/databases/:databaseId/sync-runs", async (request, reply) => {
|
||||
if (!manage(request, reply)) return reply;
|
||||
try {
|
||||
const databaseId = idSchema.parse((request.params as { databaseId?: unknown }).databaseId);
|
||||
const input = createRunSchema.parse(request.body);
|
||||
const database = await deps.repository.get(databaseId);
|
||||
if (!database) return reply.code(404).send({ code: "database_not_found", message: "Database configuration was not found." });
|
||||
if (database.version !== input.version) {
|
||||
return reply.code(409).send({ code: "database_stale", message: "Database configuration changed. Reload and try again." });
|
||||
}
|
||||
return reply.code(202).send(publicRun(await deps.worker.start(database, input.scope, input.tableIds)));
|
||||
} catch (error) { return safeError(reply, error); }
|
||||
});
|
||||
|
||||
app.get("/catalog/databases/:databaseId/sync-runs", async (request, reply) => {
|
||||
if (!manage(request, reply)) return reply;
|
||||
try {
|
||||
const databaseId = idSchema.parse((request.params as { databaseId?: unknown }).databaseId);
|
||||
if (!(await deps.repository.get(databaseId))) {
|
||||
return reply.code(404).send({ code: "database_not_found", message: "Database configuration was not found." });
|
||||
}
|
||||
return (await deps.repository.listSyncRuns(databaseId)).map(publicRun);
|
||||
} catch (error) { return safeError(reply, error); }
|
||||
});
|
||||
|
||||
app.get("/catalog/sync-runs/:runId", async (request, reply) => {
|
||||
if (!manage(request, reply)) return reply;
|
||||
try {
|
||||
const runId = idSchema.parse((request.params as { runId?: unknown }).runId);
|
||||
const run = await deps.repository.getSyncRun(runId);
|
||||
return run ? publicRun(run) : reply.code(404).send({ code: "sync_run_not_found", message: "Synchronization run was not found." });
|
||||
} catch (error) { return safeError(reply, error); }
|
||||
});
|
||||
|
||||
app.post("/catalog/sync-runs/:runId/confirm", async (request, reply) => {
|
||||
if (!manage(request, reply)) return reply;
|
||||
try {
|
||||
const runId = idSchema.parse((request.params as { runId?: unknown }).runId);
|
||||
const { confirmationToken } = confirmationSchema.parse(request.body);
|
||||
const run = await deps.worker.confirm(runId, confirmationToken);
|
||||
return run ? publicRun(run) : reply.code(404).send({ code: "sync_run_not_found", message: "Synchronization run was not found." });
|
||||
} catch (error) { return safeError(reply, error); }
|
||||
});
|
||||
|
||||
app.post("/catalog/sync-runs/:runId/cancel", async (request, reply) => {
|
||||
if (!manage(request, reply)) return reply;
|
||||
try {
|
||||
const runId = idSchema.parse((request.params as { runId?: unknown }).runId);
|
||||
const run = await deps.worker.cancel(runId);
|
||||
return run ? publicRun(run) : reply.code(404).send({ code: "sync_run_not_found", message: "Synchronization run was not found." });
|
||||
} catch (error) { return safeError(reply, error); }
|
||||
});
|
||||
|
||||
app.post("/catalog/sync-runs/:runId/retry", async (request, reply) => {
|
||||
if (!manage(request, reply)) return reply;
|
||||
try {
|
||||
const runId = idSchema.parse((request.params as { runId?: unknown }).runId);
|
||||
const run = await deps.worker.retry(runId);
|
||||
return run ? reply.code(202).send(publicRun(run)) : reply.code(404).send({ code: "sync_run_not_found", message: "Synchronization run was not found." });
|
||||
} catch (error) { return safeError(reply, error); }
|
||||
});
|
||||
|
||||
app.get("/catalog/sync-runs/:runId/events", async (request, reply) => {
|
||||
if (!manage(request, reply)) return reply;
|
||||
try {
|
||||
const runId = idSchema.parse((request.params as { runId?: unknown }).runId);
|
||||
let after = eventQuerySchema.parse(request.query).after;
|
||||
const headerCursor = Number(request.headers["last-event-id"]);
|
||||
if (Number.isInteger(headerCursor) && headerCursor >= 0) after = Math.max(after, headerCursor);
|
||||
if (!(await deps.repository.getSyncRun(runId))) {
|
||||
return reply.code(404).send({ code: "sync_run_not_found", message: "Synchronization run was not found." });
|
||||
}
|
||||
reply.hijack();
|
||||
reply.raw.writeHead(200, {
|
||||
"content-type": "text/event-stream; charset=utf-8",
|
||||
"cache-control": "no-cache, no-transform",
|
||||
connection: "keep-alive",
|
||||
"x-accel-buffering": "no",
|
||||
});
|
||||
const controller = new AbortController();
|
||||
request.raw.once("close", () => controller.abort());
|
||||
let lastRunUpdate = "";
|
||||
while (!controller.signal.aborted) {
|
||||
const events = await deps.repository.listSyncEvents(runId, after);
|
||||
for (const event of events) {
|
||||
after = event.sequence;
|
||||
reply.raw.write(`id: ${event.sequence}\nevent: log\ndata: ${JSON.stringify(event)}\n\n`);
|
||||
}
|
||||
const run = await deps.repository.getSyncRun(runId);
|
||||
if (!run) break;
|
||||
if (run.updatedAt !== lastRunUpdate) {
|
||||
lastRunUpdate = run.updatedAt;
|
||||
reply.raw.write(`event: run\ndata: ${JSON.stringify(publicRun(run))}\n\n`);
|
||||
}
|
||||
if (["succeeded", "failed", "cancelled", "interrupted"].includes(run.state)) break;
|
||||
await delay(500, undefined, { signal: controller.signal }).catch(() => undefined);
|
||||
}
|
||||
reply.raw.end();
|
||||
return reply;
|
||||
} catch (error) { return safeError(reply, error); }
|
||||
});
|
||||
|
||||
app.get("/catalog/sync-runs/:runId/events-list", async (request, reply) => {
|
||||
if (!manage(request, reply)) return reply;
|
||||
try {
|
||||
const runId = idSchema.parse((request.params as { runId?: unknown }).runId);
|
||||
const after = eventQuerySchema.parse(request.query).after;
|
||||
if (!(await deps.repository.getSyncRun(runId))) {
|
||||
return reply.code(404).send({ code: "sync_run_not_found", message: "Synchronization run was not found." });
|
||||
}
|
||||
return await deps.repository.listSyncEvents(runId, after);
|
||||
} catch (error) { return safeError(reply, error); }
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
import type { FastifyInstance, FastifyReply, FastifyRequest } from "fastify";
|
||||
import { z } from "zod";
|
||||
import { isPrincipalContext, requirePermission } from "../auth/authorization.js";
|
||||
import type { CatalogTableService } from "../catalog/table-service.js";
|
||||
import {
|
||||
CatalogConnectorError,
|
||||
CatalogOperationInProgressError,
|
||||
CatalogUnavailableError,
|
||||
type CatalogRepository,
|
||||
} from "../catalog/types.js";
|
||||
|
||||
const idSchema = z.uuid();
|
||||
const updateSchema = z.object({
|
||||
version: z.number().int().positive(),
|
||||
description: z.string().max(20_000).nullable(),
|
||||
generatedDescription: z.string().max(20_000).nullable().optional(),
|
||||
}).strict();
|
||||
|
||||
function manage(request: FastifyRequest, reply: FastifyReply) {
|
||||
return isPrincipalContext(requirePermission(request, reply, "database.manage"));
|
||||
}
|
||||
|
||||
function safeError(reply: FastifyReply, error: unknown) {
|
||||
if (error instanceof CatalogUnavailableError) {
|
||||
return reply.code(503).send({ code: "catalog_unavailable", message: "Database catalog is unavailable." });
|
||||
}
|
||||
if (error instanceof CatalogOperationInProgressError) {
|
||||
return reply.code(409).send({ code: "database_operation_in_progress", message: "A database operation is already in progress." });
|
||||
}
|
||||
if (error instanceof CatalogConnectorError) {
|
||||
return reply.code(502).send({ code: "table_introspection_failed", message: "Database tables could not be read." });
|
||||
}
|
||||
if (error instanceof z.ZodError) {
|
||||
return reply.code(400).send({ code: "table_invalid", message: "Table request is invalid." });
|
||||
}
|
||||
return reply.code(500).send({ code: "table_operation_failed", message: "Table operation failed." });
|
||||
}
|
||||
|
||||
export function catalogTableRoutes(
|
||||
app: FastifyInstance,
|
||||
deps: { repository: CatalogRepository; service: CatalogTableService },
|
||||
): void {
|
||||
app.get("/catalog/databases/:databaseId/tables", async (request, reply) => {
|
||||
if (!manage(request, reply)) return reply;
|
||||
try {
|
||||
const databaseId = idSchema.parse((request.params as { databaseId?: unknown }).databaseId);
|
||||
if (!(await deps.repository.get(databaseId))) {
|
||||
return reply.code(404).send({ code: "database_not_found", message: "Database configuration was not found." });
|
||||
}
|
||||
return await deps.service.list(databaseId);
|
||||
} catch (error) { return safeError(reply, error); }
|
||||
});
|
||||
|
||||
app.patch("/catalog/databases/:databaseId/tables/:tableId", async (request, reply) => {
|
||||
if (!manage(request, reply)) return reply;
|
||||
try {
|
||||
const params = request.params as { databaseId?: unknown; tableId?: unknown };
|
||||
const databaseId = idSchema.parse(params.databaseId);
|
||||
const tableId = idSchema.parse(params.tableId);
|
||||
const input = updateSchema.parse(request.body);
|
||||
const { version, description } = input;
|
||||
const current = await deps.repository.getTable(databaseId, tableId);
|
||||
if (!current) return reply.code(404).send({ code: "table_not_found", message: "Catalog table was not found." });
|
||||
if (current.version !== version) {
|
||||
return reply.code(409).send({ code: "table_stale", message: "Table description changed. Reload and try again." });
|
||||
}
|
||||
const updated = await deps.service.updateMetadata(
|
||||
databaseId,
|
||||
tableId,
|
||||
version,
|
||||
description,
|
||||
input.generatedDescription === undefined ? current.generatedDescription : input.generatedDescription,
|
||||
);
|
||||
if (!updated) return reply.code(409).send({ code: "table_stale", message: "Table description changed. Reload and try again." });
|
||||
return updated;
|
||||
} catch (error) { return safeError(reply, error); }
|
||||
});
|
||||
|
||||
}
|
||||
@@ -187,6 +187,7 @@ test("roles collapse duplicates and admin contains all administrative permission
|
||||
"settings.manage",
|
||||
"workspace.manage",
|
||||
"workspace.secrets.manage",
|
||||
"database.manage",
|
||||
"pi.manage",
|
||||
"auth.diagnostics.read",
|
||||
]);
|
||||
|
||||
@@ -114,7 +114,7 @@ test.each([
|
||||
const created = await fixture.app.thothiiAuthSessionStore?.create({
|
||||
principal: {
|
||||
issuer: "local", subject: user.id, displayName: user.username, roles: ["admin"],
|
||||
permissions: ["session.use", "session.read_all", "session.manage_all", "settings.manage", "workspace.manage", "workspace.secrets.manage", "pi.manage", "auth.diagnostics.read"],
|
||||
permissions: ["session.use", "session.read_all", "session.manage_all", "settings.manage", "workspace.manage", "workspace.secrets.manage", "database.manage", "pi.manage", "auth.diagnostics.read"],
|
||||
isAdmin: true,
|
||||
},
|
||||
method: "local",
|
||||
|
||||
@@ -130,7 +130,7 @@ test("local login sets a non-persistent opaque session cookie and exposes only a
|
||||
roles: ["admin"],
|
||||
permissions: [
|
||||
"session.use", "session.read_all", "session.manage_all", "settings.manage",
|
||||
"workspace.manage", "workspace.secrets.manage", "pi.manage", "auth.diagnostics.read",
|
||||
"workspace.manage", "workspace.secrets.manage", "database.manage", "pi.manage", "auth.diagnostics.read",
|
||||
],
|
||||
isAdmin: true,
|
||||
csrfToken: expect.stringMatching(/^[A-Za-z0-9_-]{43}$/),
|
||||
|
||||
@@ -32,7 +32,7 @@ test("local mode resolves a stable local principal", async () => {
|
||||
roles: ["admin"],
|
||||
permissions: [
|
||||
"session.use", "session.read_all", "session.manage_all", "settings.manage",
|
||||
"workspace.manage", "workspace.secrets.manage", "pi.manage", "auth.diagnostics.read",
|
||||
"workspace.manage", "workspace.secrets.manage", "database.manage", "pi.manage", "auth.diagnostics.read",
|
||||
],
|
||||
isAdmin: true,
|
||||
});
|
||||
@@ -74,7 +74,7 @@ test("upstream mode accepts only normalized proxy principal headers", async () =
|
||||
issuer: "portal", subject: "42", displayName: "Alice", roles: ["user", "admin"],
|
||||
permissions: [
|
||||
"session.use", "session.read_all", "session.manage_all", "settings.manage",
|
||||
"workspace.manage", "workspace.secrets.manage", "pi.manage", "auth.diagnostics.read",
|
||||
"workspace.manage", "workspace.secrets.manage", "database.manage", "pi.manage", "auth.diagnostics.read",
|
||||
],
|
||||
isAdmin: true,
|
||||
});
|
||||
|
||||
@@ -15,14 +15,14 @@ const admin: PrincipalContext = {
|
||||
issuer: "oidc", subject: "admin", roles: ["admin"],
|
||||
permissions: [
|
||||
"session.use", "session.read_all", "session.manage_all", "settings.manage",
|
||||
"workspace.manage", "workspace.secrets.manage", "pi.manage", "auth.diagnostics.read",
|
||||
"workspace.manage", "workspace.secrets.manage", "database.manage", "pi.manage", "auth.diagnostics.read",
|
||||
],
|
||||
isAdmin: true,
|
||||
};
|
||||
|
||||
const catalog: readonly Permission[] = [
|
||||
"session.use", "session.read_all", "session.manage_all", "settings.manage",
|
||||
"workspace.manage", "workspace.secrets.manage", "pi.manage", "auth.diagnostics.read",
|
||||
"workspace.manage", "workspace.secrets.manage", "database.manage", "pi.manage", "auth.diagnostics.read",
|
||||
];
|
||||
|
||||
test("permission matrix gives role-less identities no access, users session use, and admins every catalog permission", () => {
|
||||
|
||||
@@ -0,0 +1,122 @@
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { afterEach, expect, test, vi } from "vitest";
|
||||
import { buildApp } from "../src/app.js";
|
||||
import { loadConfig } from "../src/config.js";
|
||||
import { MemoryCatalogRepository } from "../src/catalog/memory-repository.js";
|
||||
import { WorkspaceSecretStore } from "../src/workspaces/secret-store.js";
|
||||
import type { WorkspaceRegistry, WorkspaceRevision } from "../src/workspaces/registry.js";
|
||||
import type { WorkspaceDescriptor } from "../src/workspaces/schema.js";
|
||||
|
||||
const roots: string[] = [];
|
||||
afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); });
|
||||
|
||||
const workspace: WorkspaceDescriptor = {
|
||||
workspace: { schema_version: 3, id: "psd-clinical", name: "Policlinico San Donato", language: "it" },
|
||||
dwh: {
|
||||
engine: "postgres", database: "warehouse", schema: "datawarehouse", port: 5432,
|
||||
supported_transports: ["postgres_direct", "rest_api"],
|
||||
},
|
||||
semantic_index: {
|
||||
vector_store: { engine: "qdrant", collection: "psd", dimensions: 1024, distance: "cosine" },
|
||||
embedding: { provider: "ollama_internal", model: "qwen3-embedding:0.6b", dimensions: 1024 },
|
||||
},
|
||||
llm_policy: { allowed: ["zai/glm-5.2"] },
|
||||
diagnostics: { dwh_rest: { method: "GET", path: "/health", auth: "bearer", response: { database: "database", schema: "schema" } } },
|
||||
};
|
||||
const revision: WorkspaceRevision = { id: "psd-clinical", commit: "a".repeat(40), blob: "b".repeat(40), snapshotPath: "/tmp/psd.yaml" };
|
||||
|
||||
function setup() {
|
||||
const secretRoot = mkdtempSync(join(tmpdir(), "catalog-secret-"));
|
||||
const runtimeRoot = mkdtempSync(join(tmpdir(), "catalog-secret-runtime-"));
|
||||
roots.push(secretRoot, runtimeRoot);
|
||||
const secretStore = new WorkspaceSecretStore({ root: secretRoot, runtimeRoot, installationId: "test" });
|
||||
const repository = new MemoryCatalogRepository();
|
||||
const registry = {
|
||||
list: vi.fn(async () => [revision]),
|
||||
listCatalog: vi.fn(async () => [{ id: "psd-clinical", name: "Policlinico San Donato", configurationState: "ready", revision }]),
|
||||
read: vi.fn(async () => ({ workspace, revision })),
|
||||
} as unknown as WorkspaceRegistry;
|
||||
const app = buildApp(loadConfig({ THT_HARNESS_DIR: "/missing", NODE_ENV: "test" }), {
|
||||
thtRunner: {} as never,
|
||||
workspaceRegistry: registry,
|
||||
workspaceSecretStore: secretStore,
|
||||
catalogRepository: repository,
|
||||
workspaceDiagnoser: vi.fn(),
|
||||
});
|
||||
return { app, secretStore, repository };
|
||||
}
|
||||
|
||||
const direct = {
|
||||
workspaceId: "psd-clinical",
|
||||
engine: "postgres",
|
||||
databaseName: "warehouse",
|
||||
schema: "datawarehouse",
|
||||
binding: { transport: "postgres_direct", host: "db.internal", port: 5432, username: "reader" },
|
||||
};
|
||||
|
||||
test("lists every YAML workspace and creates its one database configuration", async () => {
|
||||
const { app } = setup();
|
||||
const initial = await app.inject({ method: "GET", url: "/catalog/databases" });
|
||||
expect(initial.statusCode).toBe(200);
|
||||
expect(initial.json()).toMatchObject([{ workspaceId: "psd-clinical", configured: false, databaseName: "warehouse" }]);
|
||||
|
||||
const created = await app.inject({ method: "POST", url: "/catalog/databases", payload: direct });
|
||||
expect(created.statusCode).toBe(201);
|
||||
expect(created.json()).toMatchObject({ configured: true, workspaceId: "psd-clinical", version: 1 });
|
||||
expect((await app.inject({ method: "POST", url: "/catalog/databases", payload: direct })).statusCode).toBe(409);
|
||||
|
||||
const listed = await app.inject({ method: "GET", url: "/catalog/databases" });
|
||||
expect(listed.json()).toMatchObject([{ configured: true, binding: { transport: "postgres_direct", host: "db.internal" } }]);
|
||||
});
|
||||
|
||||
test("lists orphaned records and takes the REST diagnostic path from workspace YAML", async () => {
|
||||
const { app, repository } = setup();
|
||||
await repository.create({
|
||||
workspaceId: "removed-workspace",
|
||||
engine: "postgres",
|
||||
databaseName: "legacy",
|
||||
schema: "public",
|
||||
binding: { transport: "postgres_direct", host: "legacy.internal", port: 5432, username: "reader" },
|
||||
});
|
||||
const created = await app.inject({
|
||||
method: "POST",
|
||||
url: "/catalog/databases",
|
||||
payload: {
|
||||
...direct,
|
||||
binding: {
|
||||
transport: "rest_api", baseUrl: "https://psd.example/api", restPath: "/client-controlled", restAuth: "bearer",
|
||||
},
|
||||
},
|
||||
});
|
||||
expect(created.statusCode).toBe(201);
|
||||
expect(created.json()).toMatchObject({ binding: { restPath: "/health" } });
|
||||
|
||||
const rows = (await app.inject({ method: "GET", url: "/catalog/databases" })).json();
|
||||
expect(rows).toEqual(expect.arrayContaining([
|
||||
expect.objectContaining({ workspaceId: "removed-workspace", configured: true, workspaceAvailable: false }),
|
||||
expect.objectContaining({ workspaceId: "psd-clinical", configured: true, workspaceAvailable: true }),
|
||||
]));
|
||||
});
|
||||
|
||||
test("uses optimistic versions, keeps secrets write-only, and hard-deletes only local configuration", async () => {
|
||||
const { app, secretStore } = setup();
|
||||
const created = (await app.inject({ method: "POST", url: "/catalog/databases", payload: direct })).json();
|
||||
const stale = await app.inject({ method: "PATCH", url: `/catalog/databases/${created.id}`, payload: { ...direct, version: 99 } });
|
||||
expect(stale.statusCode).toBe(409);
|
||||
|
||||
const secret = await app.inject({
|
||||
method: "PUT", url: `/catalog/databases/${created.id}/secrets`,
|
||||
payload: { version: 1, values: { password: "do-not-return-this" } },
|
||||
});
|
||||
expect(secret.statusCode).toBe(200);
|
||||
expect(secret.body).not.toContain("do-not-return-this");
|
||||
expect(secret.json()).toMatchObject({ version: 2, secrets: { password: true } });
|
||||
expect(secretStore.has("psd-clinical", "catalog.dwh.password")).toBe(true);
|
||||
|
||||
const removed = await app.inject({ method: "DELETE", url: `/catalog/databases/${created.id}?version=2` });
|
||||
expect(removed.statusCode).toBe(204);
|
||||
expect(secretStore.has("psd-clinical", "catalog.dwh.password")).toBe(false);
|
||||
expect((await app.inject({ method: "GET", url: "/catalog/databases" })).json()).toMatchObject([{ configured: false }]);
|
||||
});
|
||||
@@ -0,0 +1,171 @@
|
||||
import { EventEmitter } from "node:events";
|
||||
import { existsSync, mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { PassThrough } from "node:stream";
|
||||
import type { ChildProcessWithoutNullStreams } from "node:child_process";
|
||||
import type { Client, ClientConfig } from "pg";
|
||||
import { afterEach, expect, test, vi } from "vitest";
|
||||
import {
|
||||
buildSshArguments,
|
||||
ConcreteCatalogPostgresAccess,
|
||||
} from "../src/catalog/postgres-access.js";
|
||||
import { CATALOG_SECRET_IDS } from "../src/catalog/secrets.js";
|
||||
import type { WorkspaceDatabase } from "../src/catalog/types.js";
|
||||
import { WorkspaceSecretStore } from "../src/workspaces/secret-store.js";
|
||||
|
||||
const roots: string[] = [];
|
||||
|
||||
afterEach(() => {
|
||||
for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
function secretStore() {
|
||||
const root = mkdtempSync(join(tmpdir(), "catalog-ssh-secrets-"));
|
||||
const runtimeRoot = mkdtempSync(join(tmpdir(), "catalog-ssh-runtime-"));
|
||||
roots.push(root, runtimeRoot);
|
||||
return new WorkspaceSecretStore({ root, runtimeRoot, installationId: "test" });
|
||||
}
|
||||
|
||||
function sshDatabase(): WorkspaceDatabase {
|
||||
return {
|
||||
id: "11111111-1111-4111-8111-111111111111",
|
||||
workspaceId: "psd-clinical",
|
||||
engine: "postgres",
|
||||
databaseName: "warehouse",
|
||||
schema: "datawarehouse",
|
||||
version: 4,
|
||||
createdAt: "2026-08-27T08:00:00Z",
|
||||
updatedAt: "2026-08-27T09:00:00Z",
|
||||
connectionStatus: "reachable",
|
||||
binding: {
|
||||
transport: "ssh_tunnel",
|
||||
username: "warehouse_reader",
|
||||
sshHost: "bastion.internal",
|
||||
sshPort: 2222,
|
||||
sshUsername: "tunnel_user",
|
||||
sshTargetHost: "postgres.internal",
|
||||
sshTargetPort: 5432,
|
||||
tlsServername: "postgres.internal",
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function fakeChild(): ChildProcessWithoutNullStreams {
|
||||
const child = new EventEmitter() as EventEmitter & {
|
||||
stdin: PassThrough;
|
||||
stdout: PassThrough;
|
||||
stderr: PassThrough;
|
||||
exitCode: number | null;
|
||||
signalCode: NodeJS.Signals | null;
|
||||
kill: (signal?: NodeJS.Signals | number) => boolean;
|
||||
};
|
||||
child.stdin = new PassThrough();
|
||||
child.stdout = new PassThrough();
|
||||
child.stderr = new PassThrough();
|
||||
child.exitCode = null;
|
||||
child.signalCode = null;
|
||||
child.kill = vi.fn((signal: NodeJS.Signals | number = "SIGTERM") => {
|
||||
child.signalCode = typeof signal === "string" ? signal : "SIGTERM";
|
||||
child.emit("exit", null, child.signalCode);
|
||||
return true;
|
||||
});
|
||||
return child as unknown as ChildProcessWithoutNullStreams;
|
||||
}
|
||||
|
||||
test("builds a strict host-verified OpenSSH stdio tunnel", () => {
|
||||
const args = buildSshArguments({
|
||||
sshHost: "bastion.internal",
|
||||
sshPort: 2222,
|
||||
sshUsername: "tunnel_user",
|
||||
targetHost: "postgres.internal",
|
||||
targetPort: 5432,
|
||||
privateKeyFile: "/runtime/id",
|
||||
knownHostsFile: "/runtime/known_hosts",
|
||||
passphraseFile: "/runtime/passphrase",
|
||||
connectTimeoutMs: 5_001,
|
||||
});
|
||||
|
||||
expect(args).toEqual(expect.arrayContaining([
|
||||
"-F", "/dev/null",
|
||||
"-o", "BatchMode=no",
|
||||
"-o", "StrictHostKeyChecking=yes",
|
||||
"-o", "UserKnownHostsFile=/runtime/known_hosts",
|
||||
"-o", "GlobalKnownHostsFile=/dev/null",
|
||||
"-o", "IdentitiesOnly=yes",
|
||||
"-o", "IdentityAgent=none",
|
||||
"-o", "PasswordAuthentication=no",
|
||||
"-o", "KbdInteractiveAuthentication=no",
|
||||
"-o", "ConnectTimeout=6",
|
||||
"-W", "postgres.internal:5432",
|
||||
"--", "tunnel_user@bastion.internal",
|
||||
]));
|
||||
});
|
||||
|
||||
test("connects pg through OpenSSH, supplies askpass, and releases all secret leases", async () => {
|
||||
const store = secretStore();
|
||||
store.putMany("psd-clinical", {
|
||||
[CATALOG_SECRET_IDS.password]: "db-password ",
|
||||
[CATALOG_SECRET_IDS.sshPrivateKey]: "PRIVATE KEY\n",
|
||||
[CATALOG_SECRET_IDS.sshPrivateKeyPassphrase]: "key-passphrase",
|
||||
[CATALOG_SECRET_IDS.sshKnownHosts]: "bastion.internal ssh-ed25519 AAAATEST\n",
|
||||
[CATALOG_SECRET_IDS.tlsCa]: "CA CERTIFICATE\n",
|
||||
});
|
||||
const child = fakeChild();
|
||||
let clientConfig: ClientConfig | undefined;
|
||||
let spawnCall: { command: string; args: readonly string[]; env: NodeJS.ProcessEnv } | undefined;
|
||||
const end = vi.fn(async () => undefined);
|
||||
const query = vi.fn(async () => ({ rows: [{ ok: true }] }));
|
||||
const connect = vi.fn(async () => undefined);
|
||||
|
||||
const access = new ConcreteCatalogPostgresAccess(store, {
|
||||
sshBinary: "/usr/bin/ssh",
|
||||
askpassPath: "/app/ssh-askpass.mjs",
|
||||
connectTimeoutMs: 5_000,
|
||||
spawnSsh: (command, args, options) => {
|
||||
spawnCall = { command, args, env: options.env };
|
||||
return child;
|
||||
},
|
||||
createClient: (config) => {
|
||||
clientConfig = config;
|
||||
return { connect, query, end } as unknown as Client;
|
||||
},
|
||||
});
|
||||
|
||||
const client = await access.connect(sshDatabase(), new AbortController().signal);
|
||||
expect(connect).toHaveBeenCalledOnce();
|
||||
expect(clientConfig).toMatchObject({
|
||||
host: "postgres.internal",
|
||||
port: 5432,
|
||||
database: "warehouse",
|
||||
user: "warehouse_reader",
|
||||
password: "db-password ",
|
||||
connectionTimeoutMillis: 5_000,
|
||||
ssl: {
|
||||
ca: "CA CERTIFICATE\n",
|
||||
servername: "postgres.internal",
|
||||
rejectUnauthorized: true,
|
||||
},
|
||||
});
|
||||
expect(clientConfig?.stream).toBeTypeOf("function");
|
||||
expect(spawnCall?.command).toBe("/usr/bin/ssh");
|
||||
expect(spawnCall?.args.some((argument) => argument.startsWith("IdentityFile="))).toBe(true);
|
||||
expect(spawnCall?.args.some((argument) => argument.startsWith("UserKnownHostsFile="))).toBe(true);
|
||||
expect(spawnCall?.env).toMatchObject({
|
||||
DISPLAY: "thothii",
|
||||
SSH_ASKPASS: "/app/ssh-askpass.mjs",
|
||||
SSH_ASKPASS_REQUIRE: "force",
|
||||
});
|
||||
const leasedPaths = spawnCall!.args
|
||||
.filter((argument) => argument.startsWith("IdentityFile=") || argument.startsWith("UserKnownHostsFile="))
|
||||
.map((argument) => argument.slice(argument.indexOf("=") + 1));
|
||||
leasedPaths.push(spawnCall!.env.THT_SSH_PASSPHRASE_FILE!);
|
||||
expect(leasedPaths.every(existsSync)).toBe(true);
|
||||
|
||||
await expect(client.query("SELECT 1", [])).resolves.toEqual({ rows: [{ ok: true }] });
|
||||
await client.end();
|
||||
|
||||
expect(end).toHaveBeenCalledOnce();
|
||||
expect(child.kill).toHaveBeenCalledWith("SIGTERM");
|
||||
expect(leasedPaths.some(existsSync)).toBe(false);
|
||||
});
|
||||
@@ -0,0 +1,125 @@
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { PostgreSqlContainer } from "@testcontainers/postgresql";
|
||||
import { CamelCasePlugin, Kysely, PostgresDialect, sql } from "kysely";
|
||||
import { Pool } from "pg";
|
||||
import { expect, test } from "vitest";
|
||||
import { KyselyCatalogRepository, type CatalogDatabase } from "../src/catalog/repository.js";
|
||||
import type { ObservedSchemaSnapshot } from "../src/catalog/types.js";
|
||||
import { up as upDatabases } from "../src/catalog/migrations/001_workspace_databases.js";
|
||||
import { up as upTables } from "../src/catalog/migrations/002_catalog_tables.js";
|
||||
import { up as upSchemaSync } from "../src/catalog/migrations/003_catalog_schema_sync.js";
|
||||
import { up as upRuntimeSequencePrivileges } from "../src/catalog/migrations/004_catalog_runtime_sequence_privileges.js";
|
||||
|
||||
const dockerAvailable = spawnSync("docker", ["info"], { stdio: "ignore" }).status === 0;
|
||||
|
||||
test.skipIf(!dockerAvailable)("PostgreSQL migration enforces one database per workspace and optimistic updates", async () => {
|
||||
const container = await new PostgreSqlContainer("postgres:17.6-bookworm").start();
|
||||
const db = new Kysely<CatalogDatabase>({
|
||||
dialect: new PostgresDialect({ pool: new Pool({ connectionString: container.getConnectionUri() }) }),
|
||||
plugins: [new CamelCasePlugin()],
|
||||
});
|
||||
try {
|
||||
await upDatabases(db);
|
||||
await upTables(db);
|
||||
await upSchemaSync(db);
|
||||
await sql`CREATE ROLE thothii_catalog_runtime`.execute(db);
|
||||
await upRuntimeSequencePrivileges(db);
|
||||
const sequencePrivilege = await sql<{ allowed: boolean }>`
|
||||
SELECT has_sequence_privilege(
|
||||
'thothii_catalog_runtime',
|
||||
'catalog_sync_events_id_seq',
|
||||
'USAGE'
|
||||
) AS allowed
|
||||
`.execute(db);
|
||||
expect(sequencePrivilege.rows[0]?.allowed).toBe(true);
|
||||
const repository = new KyselyCatalogRepository(db);
|
||||
const input = {
|
||||
workspaceId: "psd-clinical",
|
||||
engine: "postgres" as const,
|
||||
databaseName: "warehouse",
|
||||
schema: "datawarehouse",
|
||||
binding: { transport: "rest_api" as const, baseUrl: "https://psd.example/api", restPath: "/health", restAuth: "bearer" as const },
|
||||
};
|
||||
const created = await repository.create(input);
|
||||
expect(created).toMatchObject({ version: 1, connectionStatus: "untested", binding: { transport: "rest_api" } });
|
||||
await expect(repository.create(input)).rejects.toThrow("Workspace database already exists");
|
||||
expect(await repository.update(created.id, 99, input)).toBeUndefined();
|
||||
const synchronized = await repository.reconcileTables(created.id, 1, [
|
||||
{ name: "patients", sourceComment: "Clinical patients" },
|
||||
{ name: "visits", sourceComment: null },
|
||||
], []);
|
||||
expect(synchronized).toMatchObject({ kind: "applied", createdCount: 2, deletedCount: 0 });
|
||||
const patients = (await repository.listTables(created.id))[0];
|
||||
expect(await repository.updateTableDescription(
|
||||
created.id,
|
||||
patients.id,
|
||||
patients.version,
|
||||
"Curated patients",
|
||||
)).toMatchObject({ description: "Curated patients", sourceComment: "Clinical patients", version: 2 });
|
||||
const visits = (await repository.listTables(created.id)).find((table) => table.name === "visits")!;
|
||||
const fullColumnsSnapshot: ObservedSchemaSnapshot = {
|
||||
schemaVersion: 1,
|
||||
capabilities: { tables: "available", columns: "available", relationships: "available" },
|
||||
tables: [
|
||||
{ name: "patients", sourceComment: "Clinical patients" },
|
||||
{ name: "visits", sourceComment: null },
|
||||
],
|
||||
columns: [
|
||||
{ tableName: "patients", name: "id", ordinalPosition: 1, dataType: "bigint", isNullable: false, defaultExpression: null, primaryKeyPosition: 1, sourceComment: null },
|
||||
{ tableName: "patients", name: "name", ordinalPosition: 2, dataType: "text", isNullable: true, defaultExpression: null, primaryKeyPosition: null, sourceComment: null },
|
||||
{ tableName: "visits", name: "id", ordinalPosition: 1, dataType: "bigint", isNullable: false, defaultExpression: null, primaryKeyPosition: 1, sourceComment: null },
|
||||
{ tableName: "visits", name: "patient_id", ordinalPosition: 2, dataType: "bigint", isNullable: false, defaultExpression: null, primaryKeyPosition: null, sourceComment: null },
|
||||
],
|
||||
relationships: [],
|
||||
};
|
||||
expect(await repository.applySchemaSync(created.id, 1, "columns", [], fullColumnsSnapshot))
|
||||
.toMatchObject({ created: 4, deleted: 0 });
|
||||
expect((await repository.listColumns(created.id, patients.id)).map((column) => column.name))
|
||||
.toEqual(["id", "name"]);
|
||||
expect((await repository.listColumns(created.id, visits.id)).map((column) => column.name))
|
||||
.toEqual(["id", "patient_id"]);
|
||||
|
||||
const reducedColumnsSnapshot: ObservedSchemaSnapshot = {
|
||||
...fullColumnsSnapshot,
|
||||
columns: fullColumnsSnapshot.columns.filter((column) => column.name === "id"),
|
||||
};
|
||||
expect(await repository.planSchemaSync(created.id, "columns", [], reducedColumnsSnapshot)).toMatchObject({
|
||||
deletedColumns: [
|
||||
{ tableName: "patients", columnName: "name" },
|
||||
{ tableName: "visits", columnName: "patient_id" },
|
||||
],
|
||||
});
|
||||
expect(await repository.planSchemaSync(created.id, "columns", [patients.id], reducedColumnsSnapshot)).toMatchObject({
|
||||
deletedColumns: [{ tableName: "patients", columnName: "name" }],
|
||||
});
|
||||
expect(await repository.applySchemaSync(created.id, 1, "columns", [patients.id], reducedColumnsSnapshot))
|
||||
.toMatchObject({ deleted: 1 });
|
||||
expect((await repository.listColumns(created.id, patients.id)).map((column) => column.name))
|
||||
.toEqual(["id"]);
|
||||
expect((await repository.listColumns(created.id, visits.id)).map((column) => column.name))
|
||||
.toEqual(["id", "patient_id"]);
|
||||
expect(await repository.reconcileTables(created.id, 1, [
|
||||
{ name: "patients", sourceComment: "Updated physical comment" },
|
||||
], [])).toEqual({ kind: "confirmation_required", deletedNames: ["visits"] });
|
||||
expect(await repository.reconcileTables(created.id, 1, [
|
||||
{ name: "patients", sourceComment: "Updated physical comment" },
|
||||
], ["visits"])).toMatchObject({ kind: "applied", updatedCount: 1, deletedCount: 1 });
|
||||
const syncRun = await repository.createSyncRun(created.id, "columns", [patients.id], 1);
|
||||
expect(syncRun).toMatchObject({
|
||||
databaseId: created.id,
|
||||
scope: "columns",
|
||||
tableIds: [patients.id],
|
||||
state: "queued",
|
||||
});
|
||||
expect(await repository.listSyncRuns(created.id)).toEqual([
|
||||
expect.objectContaining({ id: syncRun.id, tableIds: [patients.id] }),
|
||||
]);
|
||||
expect(await repository.update(created.id, 1, { ...input, schema: "public" })).toMatchObject({ version: 2, schema: "public" });
|
||||
expect(await repository.delete(created.id, 2)).toBe(true);
|
||||
expect(await repository.list()).toEqual([]);
|
||||
expect(await repository.listTables(created.id)).toEqual([]);
|
||||
} finally {
|
||||
await db.destroy();
|
||||
await container.stop();
|
||||
}
|
||||
}, 60_000);
|
||||
@@ -0,0 +1,194 @@
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { afterEach, expect, test, vi } from "vitest";
|
||||
import type { CatalogDatabaseClient, CatalogPostgresAccess } from "../src/catalog/postgres-access.js";
|
||||
import { ConcreteCatalogSchemaIntrospector } from "../src/catalog/schema-introspector.js";
|
||||
import {
|
||||
CatalogSchemaCapabilityUnavailableError,
|
||||
type WorkspaceDatabase,
|
||||
} from "../src/catalog/types.js";
|
||||
import { WorkspaceSecretStore } from "../src/workspaces/secret-store.js";
|
||||
|
||||
const roots: string[] = [];
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals();
|
||||
for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
function store() {
|
||||
const root = mkdtempSync(join(tmpdir(), "catalog-schema-introspection-secrets-"));
|
||||
const runtimeRoot = mkdtempSync(join(tmpdir(), "catalog-schema-introspection-runtime-"));
|
||||
roots.push(root, runtimeRoot);
|
||||
return new WorkspaceSecretStore({ root, runtimeRoot, installationId: "test" });
|
||||
}
|
||||
|
||||
function database(binding: WorkspaceDatabase["binding"]): WorkspaceDatabase {
|
||||
return {
|
||||
id: "11111111-1111-4111-8111-111111111111",
|
||||
workspaceId: "psd-clinical",
|
||||
engine: "postgres",
|
||||
databaseName: "warehouse",
|
||||
schema: "datawarehouse",
|
||||
binding,
|
||||
version: 4,
|
||||
connectionStatus: "reachable",
|
||||
testedVersion: 4,
|
||||
createdAt: "2026-08-27T08:00:00Z",
|
||||
updatedAt: "2026-08-27T09:00:00Z",
|
||||
};
|
||||
}
|
||||
|
||||
test("reads columns, ordered composite keys, and physical relationships from one PostgreSQL connection", async () => {
|
||||
const query = vi.fn()
|
||||
.mockResolvedValueOnce({ rows: [{ present: true }] })
|
||||
.mockResolvedValueOnce({ rows: [{ name: "visits", source_comment: "Visits" }] })
|
||||
.mockResolvedValueOnce({ rows: [
|
||||
{ table_name: "visits", name: "tenant_id", ordinal_position: 1, data_type: "uuid", is_nullable: false, default_expression: null, primary_key_position: 1, source_comment: null },
|
||||
{ table_name: "visits", name: "patient_id", ordinal_position: 2, data_type: "bigint", is_nullable: false, default_expression: null, primary_key_position: 2, source_comment: "Patient" },
|
||||
] })
|
||||
.mockResolvedValueOnce({ rows: [
|
||||
{ constraint_name: "visits_patient_fkey", source_table_name: "visits", target_table_name: "patients", update_action: "a", delete_action: "c", deferrable: true, initially_deferred: false, position: 1, source_column_name: "tenant_id", target_column_name: "tenant_id" },
|
||||
{ constraint_name: "visits_patient_fkey", source_table_name: "visits", target_table_name: "patients", update_action: "a", delete_action: "c", deferrable: true, initially_deferred: false, position: 2, source_column_name: "patient_id", target_column_name: "id" },
|
||||
] });
|
||||
const end = vi.fn(async () => undefined);
|
||||
const client: CatalogDatabaseClient = { query, end };
|
||||
const postgres: CatalogPostgresAccess = { connect: vi.fn(async () => client) };
|
||||
const introspector = new ConcreteCatalogSchemaIntrospector(postgres, store());
|
||||
|
||||
const result = await introspector.scan(database({
|
||||
transport: "ssh_tunnel",
|
||||
username: "reader",
|
||||
sshHost: "bastion.internal",
|
||||
sshPort: 22,
|
||||
sshUsername: "tunnel",
|
||||
sshTargetHost: "db.internal",
|
||||
sshTargetPort: 5432,
|
||||
}), new AbortController().signal);
|
||||
|
||||
expect(result.capabilities).toEqual({ tables: "available", columns: "available", relationships: "available" });
|
||||
expect(result.columns).toMatchObject([
|
||||
{ name: "tenant_id", primaryKeyPosition: 1, isNullable: false },
|
||||
{ name: "patient_id", primaryKeyPosition: 2, sourceComment: "Patient" },
|
||||
]);
|
||||
expect(result.relationships).toEqual([expect.objectContaining({
|
||||
constraintName: "visits_patient_fkey",
|
||||
updateRule: "NO ACTION",
|
||||
deleteRule: "CASCADE",
|
||||
deferrable: true,
|
||||
columns: [
|
||||
{ position: 1, sourceColumnName: "tenant_id", targetColumnName: "tenant_id" },
|
||||
{ position: 2, sourceColumnName: "patient_id", targetColumnName: "id" },
|
||||
],
|
||||
})]);
|
||||
expect(query.mock.calls[2][0]).toContain("format_type");
|
||||
expect(query.mock.calls[3][0]).toContain("WITH ORDINALITY");
|
||||
expect(query.mock.calls.slice(1).every((call) => call[1][0] === "datawarehouse")).toBe(true);
|
||||
expect(end).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
test("uses the typed full REST snapshot RPC and preserves explicit capability unavailability", async () => {
|
||||
const response = {
|
||||
schemaVersion: 1,
|
||||
capabilities: { tables: "available", columns: "unavailable", relationships: "unavailable" },
|
||||
tables: [{ name: "patients", sourceComment: null }],
|
||||
columns: [],
|
||||
relationships: [],
|
||||
};
|
||||
const fetchMock = vi.fn(async () => new Response(JSON.stringify(response), { status: 200, headers: { "content-type": "application/json" } }));
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
const postgres: CatalogPostgresAccess = { connect: vi.fn(async () => { throw new Error("wire access must not be used"); }) };
|
||||
const introspector = new ConcreteCatalogSchemaIntrospector(postgres, store());
|
||||
|
||||
const result = await introspector.scan(database({
|
||||
transport: "rest_api", baseUrl: "https://connector.internal/api/", restPath: "/health", restAuth: "none",
|
||||
}), new AbortController().signal);
|
||||
|
||||
expect(result).toEqual(response);
|
||||
expect(fetchMock).toHaveBeenCalledWith(
|
||||
"https://connector.internal/api/rpc/schema_snapshot",
|
||||
expect.objectContaining({ method: "POST", body: JSON.stringify({ schema_name: "datawarehouse" }) }),
|
||||
);
|
||||
});
|
||||
|
||||
test("falls back to one read-only REST query when the snapshot RPC is absent", async () => {
|
||||
const response = {
|
||||
schemaVersion: 1 as const,
|
||||
capabilities: { tables: "available" as const, columns: "available" as const, relationships: "available" as const },
|
||||
tables: [
|
||||
{ name: "patients", sourceComment: "Clinical patients" },
|
||||
{ name: "visits", sourceComment: null },
|
||||
],
|
||||
columns: [
|
||||
{ tableName: "patients", name: "tenant_id", ordinalPosition: 1, dataType: "uuid", isNullable: false, defaultExpression: null, primaryKeyPosition: 1, sourceComment: "Tenant" },
|
||||
{ tableName: "patients", name: "id", ordinalPosition: 2, dataType: "bigint", isNullable: false, defaultExpression: "nextval('patients_id_seq'::regclass)", primaryKeyPosition: 2, sourceComment: null },
|
||||
{ tableName: "visits", name: "tenant_id", ordinalPosition: 1, dataType: "uuid", isNullable: false, defaultExpression: null, primaryKeyPosition: null, sourceComment: null },
|
||||
{ tableName: "visits", name: "patient_id", ordinalPosition: 2, dataType: "bigint", isNullable: true, defaultExpression: null, primaryKeyPosition: null, sourceComment: "Owning patient" },
|
||||
],
|
||||
relationships: [{
|
||||
constraintName: "visits_patient_fkey",
|
||||
sourceTableName: "visits",
|
||||
targetTableName: "patients",
|
||||
updateRule: "CASCADE",
|
||||
deleteRule: "RESTRICT",
|
||||
deferrable: true,
|
||||
initiallyDeferred: false,
|
||||
columns: [
|
||||
{ position: 1, sourceColumnName: "tenant_id", targetColumnName: "tenant_id" },
|
||||
{ position: 2, sourceColumnName: "patient_id", targetColumnName: "id" },
|
||||
],
|
||||
}],
|
||||
};
|
||||
const fetchMock = vi.fn()
|
||||
.mockResolvedValueOnce(new Response(null, { status: 404 }))
|
||||
.mockResolvedValueOnce(new Response(JSON.stringify([response]), {
|
||||
status: 200,
|
||||
headers: { "content-type": "application/json" },
|
||||
}));
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
const postgres: CatalogPostgresAccess = {
|
||||
connect: vi.fn(async () => { throw new Error("wire access must not be used"); }),
|
||||
};
|
||||
const introspector = new ConcreteCatalogSchemaIntrospector(postgres, store());
|
||||
|
||||
const result = await introspector.scan(database({
|
||||
transport: "rest_api",
|
||||
baseUrl: "https://connector.internal/api/",
|
||||
restPath: "/health",
|
||||
restAuth: "none",
|
||||
}), new AbortController().signal);
|
||||
|
||||
expect(result).toEqual(response);
|
||||
expect(fetchMock).toHaveBeenCalledTimes(2);
|
||||
expect(fetchMock.mock.calls[0]).toEqual([
|
||||
"https://connector.internal/api/rpc/schema_snapshot",
|
||||
expect.objectContaining({ method: "POST", body: JSON.stringify({ schema_name: "datawarehouse" }) }),
|
||||
]);
|
||||
expect(fetchMock.mock.calls[1][0]).toBe("https://connector.internal/api/rpc/run_query");
|
||||
const fallbackRequest = fetchMock.mock.calls[1][1] as RequestInit;
|
||||
expect(fallbackRequest).toMatchObject({ method: "POST" });
|
||||
const fallbackBody = JSON.parse(String(fallbackRequest.body)) as { query_text: string };
|
||||
expect(Object.keys(fallbackBody)).toEqual(["query_text"]);
|
||||
expect(fallbackBody.query_text).toMatch(/^\s*WITH\b/);
|
||||
expect(fallbackBody.query_text).toContain("pg_catalog.pg_constraint");
|
||||
expect(fallbackBody.query_text).not.toMatch(/\b(INSERT|UPDATE|DROP|ALTER|CREATE|TRUNCATE)\b/i);
|
||||
});
|
||||
|
||||
test("classifies a missing REST snapshot RPC as an explicit binding capability", async () => {
|
||||
vi.stubGlobal("fetch", vi.fn(async () => new Response(null, { status: 404 })));
|
||||
const postgres: CatalogPostgresAccess = {
|
||||
connect: vi.fn(async () => { throw new Error("wire access must not be used"); }),
|
||||
};
|
||||
const introspector = new ConcreteCatalogSchemaIntrospector(postgres, store());
|
||||
|
||||
const scan = introspector.scan(database({
|
||||
transport: "rest_api",
|
||||
baseUrl: "https://connector.internal/api/",
|
||||
restPath: "/health",
|
||||
restAuth: "none",
|
||||
}), new AbortController().signal);
|
||||
|
||||
await expect(scan).rejects.toMatchObject<CatalogSchemaCapabilityUnavailableError>({
|
||||
capability: "schema_snapshot",
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,224 @@
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { afterEach, expect, test, vi } from "vitest";
|
||||
import { buildApp } from "../src/app.js";
|
||||
import { loadConfig } from "../src/config.js";
|
||||
import { MemoryCatalogRepository } from "../src/catalog/memory-repository.js";
|
||||
import type { CatalogSchemaIntrospector } from "../src/catalog/schema-introspector.js";
|
||||
import type { CatalogSyncRun, ObservedSchemaSnapshot } from "../src/catalog/types.js";
|
||||
import { WorkspaceSecretStore } from "../src/workspaces/secret-store.js";
|
||||
import type { WorkspaceRegistry, WorkspaceRevision } from "../src/workspaces/registry.js";
|
||||
import type { WorkspaceDescriptor } from "../src/workspaces/schema.js";
|
||||
|
||||
const roots: string[] = [];
|
||||
afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); });
|
||||
|
||||
const workspace: WorkspaceDescriptor = {
|
||||
workspace: { schema_version: 3, id: "psd-clinical", name: "Policlinico San Donato", language: "it" },
|
||||
dwh: { engine: "postgres", database: "warehouse", schema: "datawarehouse", port: 5432, supported_transports: ["postgres_direct"] },
|
||||
semantic_index: {
|
||||
vector_store: { engine: "qdrant", collection: "psd", dimensions: 1024, distance: "cosine" },
|
||||
embedding: { provider: "ollama_internal", model: "qwen3-embedding:0.6b", dimensions: 1024 },
|
||||
},
|
||||
llm_policy: { allowed: ["zai/glm-5.2"] },
|
||||
};
|
||||
const revision: WorkspaceRevision = { id: "psd-clinical", commit: "a".repeat(40), blob: "b".repeat(40), snapshotPath: "/tmp/psd.yaml" };
|
||||
|
||||
function snapshot(): ObservedSchemaSnapshot {
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
capabilities: { tables: "available", columns: "available", relationships: "available" },
|
||||
tables: [
|
||||
{ name: "patients", sourceComment: "Clinical patients" },
|
||||
{ name: "visits", sourceComment: "Patient visits" },
|
||||
],
|
||||
columns: [
|
||||
{ tableName: "patients", name: "id", ordinalPosition: 1, dataType: "bigint", isNullable: false, defaultExpression: null, primaryKeyPosition: 1, sourceComment: "Patient key" },
|
||||
{ tableName: "visits", name: "id", ordinalPosition: 1, dataType: "bigint", isNullable: false, defaultExpression: null, primaryKeyPosition: 1, sourceComment: null },
|
||||
{ tableName: "visits", name: "patient_id", ordinalPosition: 2, dataType: "bigint", isNullable: false, defaultExpression: null, primaryKeyPosition: null, sourceComment: "Owning patient" },
|
||||
],
|
||||
relationships: [{
|
||||
constraintName: "visits_patient_id_fkey",
|
||||
sourceTableName: "visits",
|
||||
targetTableName: "patients",
|
||||
updateRule: "NO ACTION",
|
||||
deleteRule: "CASCADE",
|
||||
deferrable: false,
|
||||
initiallyDeferred: false,
|
||||
columns: [{ position: 1, sourceColumnName: "patient_id", targetColumnName: "id" }],
|
||||
}],
|
||||
};
|
||||
}
|
||||
|
||||
async function waitFor(repository: MemoryCatalogRepository, runId: string, state: CatalogSyncRun["state"]): Promise<CatalogSyncRun> {
|
||||
for (let attempt = 0; attempt < 100; attempt += 1) {
|
||||
const run = await repository.getSyncRun(runId);
|
||||
if (run?.state === state) return run;
|
||||
await new Promise((resolve) => setTimeout(resolve, 5));
|
||||
}
|
||||
throw new Error(`Run ${runId} did not reach ${state}`);
|
||||
}
|
||||
|
||||
async function setup() {
|
||||
const secretRoot = mkdtempSync(join(tmpdir(), "catalog-schema-secret-"));
|
||||
const runtimeRoot = mkdtempSync(join(tmpdir(), "catalog-schema-runtime-"));
|
||||
roots.push(secretRoot, runtimeRoot);
|
||||
const repository = new MemoryCatalogRepository();
|
||||
const created = await repository.create({
|
||||
workspaceId: "psd-clinical", engine: "postgres", databaseName: "warehouse", schema: "datawarehouse",
|
||||
binding: { transport: "postgres_direct", host: "db.internal", port: 5432, username: "reader" },
|
||||
});
|
||||
await repository.recordTest(created.id, created.version, {
|
||||
connectionStatus: "reachable", testedVersion: created.version, lastTestedAt: new Date().toISOString(),
|
||||
});
|
||||
let observed = snapshot();
|
||||
const scan = vi.fn(async (_database, _signal, progress) => {
|
||||
await progress?.("connecting");
|
||||
await progress?.("scanning_tables", { tables: observed.tables.length });
|
||||
await progress?.("scanning_columns", { tables: observed.tables.length, columns: observed.columns.length });
|
||||
await progress?.("scanning_relationships", { relationships: observed.relationships.length });
|
||||
return structuredClone(observed);
|
||||
});
|
||||
const introspector: CatalogSchemaIntrospector = { scan };
|
||||
const registry = {
|
||||
list: vi.fn(async () => [revision]),
|
||||
listCatalog: vi.fn(async () => [{ id: "psd-clinical", name: "Policlinico San Donato", configurationState: "ready", revision }]),
|
||||
read: vi.fn(async () => ({ workspace, revision })),
|
||||
} as unknown as WorkspaceRegistry;
|
||||
const app = buildApp(loadConfig({ THT_HARNESS_DIR: "/missing", NODE_ENV: "test" }), {
|
||||
thtRunner: {} as never,
|
||||
workspaceRegistry: registry,
|
||||
workspaceSecretStore: new WorkspaceSecretStore({ root: secretRoot, runtimeRoot, installationId: "test" }),
|
||||
catalogRepository: repository,
|
||||
catalogSchemaIntrospector: introspector,
|
||||
workspaceDiagnoser: vi.fn(),
|
||||
});
|
||||
return {
|
||||
app, repository, database: (await repository.get(created.id))!, scan,
|
||||
setObserved(next: ObservedSchemaSnapshot) { observed = next; },
|
||||
};
|
||||
}
|
||||
|
||||
test("synchronizes a full physical schema and derives primary and foreign key flags", async () => {
|
||||
const { app, repository, database } = await setup();
|
||||
const started = await app.inject({
|
||||
method: "POST", url: `/catalog/databases/${database.id}/sync-runs`,
|
||||
payload: { version: database.version, scope: "all", tableIds: [] },
|
||||
});
|
||||
expect(started.statusCode).toBe(202);
|
||||
const completed = await waitFor(repository, started.json().id, "succeeded");
|
||||
expect(completed.counts).toMatchObject({ tables: 2, columns: 3, relationships: 1 });
|
||||
|
||||
const tables = await repository.listTables(database.id);
|
||||
const visits = tables.find((table) => table.name === "visits")!;
|
||||
const columns = (await app.inject({ method: "GET", url: `/catalog/databases/${database.id}/tables/${visits.id}/columns` })).json();
|
||||
expect(columns).toMatchObject([
|
||||
{ name: "id", isPrimaryKey: true, primaryKeyPosition: 1, isForeignKey: false },
|
||||
{ name: "patient_id", isPrimaryKey: false, isForeignKey: true, foreignKeyCount: 1 },
|
||||
]);
|
||||
const relationships = (await app.inject({ method: "GET", url: `/catalog/databases/${database.id}/relationships` })).json();
|
||||
expect(relationships).toMatchObject([{ constraintName: "visits_patient_id_fkey", columns: [{ sourceColumnName: "patient_id", targetColumnName: "id" }] }]);
|
||||
expect((await repository.get(database.id))?.schemaSyncedVersion).toBe(database.version);
|
||||
});
|
||||
|
||||
test("synchronizes columns for every catalog table when no table selection is supplied", async () => {
|
||||
const { app, repository, database, setObserved } = await setup();
|
||||
const tablesRun = await app.inject({
|
||||
method: "POST",
|
||||
url: `/catalog/databases/${database.id}/sync-runs`,
|
||||
payload: { version: database.version, scope: "tables", tableIds: [] },
|
||||
});
|
||||
expect(tablesRun.statusCode).toBe(202);
|
||||
await waitFor(repository, tablesRun.json().id, "succeeded");
|
||||
const tables = await repository.listTables(database.id);
|
||||
expect(tables.map((table) => table.name)).toEqual(["patients", "visits"]);
|
||||
|
||||
const columnsRun = await app.inject({
|
||||
method: "POST",
|
||||
url: `/catalog/databases/${database.id}/sync-runs`,
|
||||
payload: { version: database.version, scope: "columns", tableIds: [] },
|
||||
});
|
||||
expect(columnsRun.statusCode).toBe(202);
|
||||
await waitFor(repository, columnsRun.json().id, "succeeded");
|
||||
const patients = tables.find((table) => table.name === "patients")!;
|
||||
const visits = tables.find((table) => table.name === "visits")!;
|
||||
expect((await repository.listColumns(database.id, patients.id)).map((column) => column.name)).toEqual(["id"]);
|
||||
expect((await repository.listColumns(database.id, visits.id)).map((column) => column.name)).toEqual(["id", "patient_id"]);
|
||||
|
||||
const next = snapshot();
|
||||
next.columns = next.columns.filter((column) => column.name !== "id");
|
||||
setObserved(next);
|
||||
const selectedDestructiveRun = await app.inject({
|
||||
method: "POST",
|
||||
url: `/catalog/databases/${database.id}/sync-runs`,
|
||||
payload: { version: database.version, scope: "columns", tableIds: [patients.id] },
|
||||
});
|
||||
expect(selectedDestructiveRun.statusCode).toBe(202);
|
||||
const selectedWaiting = await waitFor(repository, selectedDestructiveRun.json().id, "awaiting_confirmation");
|
||||
expect(selectedWaiting.plannedDiff?.deletedColumns).toEqual([
|
||||
{ tableName: "patients", columnName: "id" },
|
||||
]);
|
||||
expect((await app.inject({
|
||||
method: "POST",
|
||||
url: `/catalog/sync-runs/${selectedWaiting.id}/cancel`,
|
||||
})).statusCode).toBe(200);
|
||||
|
||||
const destructiveRun = await app.inject({
|
||||
method: "POST",
|
||||
url: `/catalog/databases/${database.id}/sync-runs`,
|
||||
payload: { version: database.version, scope: "columns", tableIds: [] },
|
||||
});
|
||||
expect(destructiveRun.statusCode).toBe(202);
|
||||
const waiting = await waitFor(repository, destructiveRun.json().id, "awaiting_confirmation");
|
||||
expect(waiting.plannedDiff?.deletedColumns).toEqual([
|
||||
{ tableName: "patients", columnName: "id" },
|
||||
{ tableName: "visits", columnName: "id" },
|
||||
]);
|
||||
});
|
||||
|
||||
test("keeps generated descriptions editable and preserves them across synchronization", async () => {
|
||||
const { app, repository, database } = await setup();
|
||||
const first = await app.inject({ method: "POST", url: `/catalog/databases/${database.id}/sync-runs`, payload: { version: database.version, scope: "all", tableIds: [] } });
|
||||
await waitFor(repository, first.json().id, "succeeded");
|
||||
const patients = (await repository.listTables(database.id)).find((table) => table.name === "patients")!;
|
||||
const editedTable = await app.inject({
|
||||
method: "PATCH", url: `/catalog/databases/${database.id}/tables/${patients.id}`,
|
||||
payload: { version: patients.version, description: null, generatedDescription: "Generated table draft" },
|
||||
});
|
||||
expect(editedTable.json()).toMatchObject({ description: null, generatedDescription: "Generated table draft" });
|
||||
const idColumn = (await repository.listColumns(database.id, patients.id))[0];
|
||||
const editedColumn = await app.inject({
|
||||
method: "PATCH", url: `/catalog/databases/${database.id}/tables/${patients.id}/columns/${idColumn.id}`,
|
||||
payload: { version: idColumn.version, description: "Reviewed key", generatedDescription: "Generated key draft" },
|
||||
});
|
||||
expect(editedColumn.json()).toMatchObject({ description: "Reviewed key", generatedDescription: "Generated key draft" });
|
||||
|
||||
const second = await app.inject({ method: "POST", url: `/catalog/databases/${database.id}/sync-runs`, payload: { version: database.version, scope: "all", tableIds: [] } });
|
||||
await waitFor(repository, second.json().id, "succeeded");
|
||||
expect(await repository.getTable(database.id, patients.id)).toMatchObject({ generatedDescription: "Generated table draft" });
|
||||
expect(await repository.getColumn(database.id, patients.id, idColumn.id)).toMatchObject({ description: "Reviewed key", generatedDescription: "Generated key draft" });
|
||||
});
|
||||
|
||||
test("waits for confirmation and rescans before applying destructive changes", async () => {
|
||||
const { app, repository, database, scan, setObserved } = await setup();
|
||||
const first = await app.inject({ method: "POST", url: `/catalog/databases/${database.id}/sync-runs`, payload: { version: database.version, scope: "all", tableIds: [] } });
|
||||
await waitFor(repository, first.json().id, "succeeded");
|
||||
const next = snapshot();
|
||||
next.tables = next.tables.filter((table) => table.name !== "visits");
|
||||
next.columns = next.columns.filter((column) => column.tableName !== "visits");
|
||||
next.relationships = [];
|
||||
setObserved(next);
|
||||
|
||||
const destructive = await app.inject({ method: "POST", url: `/catalog/databases/${database.id}/sync-runs`, payload: { version: database.version, scope: "all", tableIds: [] } });
|
||||
const waiting = await waitFor(repository, destructive.json().id, "awaiting_confirmation");
|
||||
expect(waiting.plannedDiff).toMatchObject({ deletedTables: ["visits"] });
|
||||
expect(await repository.listTables(database.id)).toHaveLength(2);
|
||||
const confirmed = await app.inject({
|
||||
method: "POST", url: `/catalog/sync-runs/${waiting.id}/confirm`, payload: { confirmationToken: waiting.confirmationToken },
|
||||
});
|
||||
expect(confirmed.statusCode).toBe(200);
|
||||
await waitFor(repository, waiting.id, "succeeded");
|
||||
expect((await repository.listTables(database.id)).map((table) => table.name)).toEqual(["patients"]);
|
||||
expect(scan).toHaveBeenCalledTimes(3);
|
||||
});
|
||||
@@ -0,0 +1,124 @@
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { afterEach, expect, test, vi } from "vitest";
|
||||
import type {
|
||||
CatalogDatabaseClient,
|
||||
CatalogPostgresAccess,
|
||||
} from "../src/catalog/postgres-access.js";
|
||||
import { CATALOG_SECRET_IDS } from "../src/catalog/secrets.js";
|
||||
import { ConcreteCatalogTableIntrospector } from "../src/catalog/table-introspector.js";
|
||||
import type { WorkspaceDatabase } from "../src/catalog/types.js";
|
||||
import { WorkspaceSecretStore } from "../src/workspaces/secret-store.js";
|
||||
|
||||
const roots: string[] = [];
|
||||
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals();
|
||||
for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
function secretStore() {
|
||||
const root = mkdtempSync(join(tmpdir(), "catalog-table-introspection-secrets-"));
|
||||
const runtimeRoot = mkdtempSync(join(tmpdir(), "catalog-table-introspection-runtime-"));
|
||||
roots.push(root, runtimeRoot);
|
||||
return new WorkspaceSecretStore({ root, runtimeRoot, installationId: "test" });
|
||||
}
|
||||
|
||||
function database(binding: WorkspaceDatabase["binding"]): WorkspaceDatabase {
|
||||
return {
|
||||
id: "11111111-1111-4111-8111-111111111111",
|
||||
workspaceId: "psd-clinical",
|
||||
engine: "postgres",
|
||||
databaseName: "warehouse",
|
||||
schema: "datawarehouse",
|
||||
version: 4,
|
||||
createdAt: "2026-08-27T08:00:00Z",
|
||||
updatedAt: "2026-08-27T09:00:00Z",
|
||||
connectionStatus: "reachable",
|
||||
binding,
|
||||
};
|
||||
}
|
||||
|
||||
test("reads only ordinary and partitioned PostgreSQL tables from the configured schema", async () => {
|
||||
const query = vi.fn()
|
||||
.mockResolvedValueOnce({ rows: [{ present: true }] })
|
||||
.mockResolvedValueOnce({ rows: [
|
||||
{ name: "visits", source_comment: null },
|
||||
{ name: "patients", source_comment: "Clinical patients" },
|
||||
] });
|
||||
const end = vi.fn(async () => undefined);
|
||||
const client: CatalogDatabaseClient = { query, end };
|
||||
const postgres: CatalogPostgresAccess = { connect: vi.fn(async () => client) };
|
||||
const introspector = new ConcreteCatalogTableIntrospector(postgres, secretStore());
|
||||
|
||||
const tables = await introspector.scan(database({
|
||||
transport: "postgres_direct",
|
||||
host: "db.internal",
|
||||
port: 5432,
|
||||
username: "reader",
|
||||
}), new AbortController().signal);
|
||||
|
||||
expect(tables).toEqual([
|
||||
{ name: "patients", sourceComment: "Clinical patients" },
|
||||
{ name: "visits", sourceComment: null },
|
||||
]);
|
||||
expect(query.mock.calls[1][0]).toContain("c.relkind IN ('r', 'p')");
|
||||
expect(query.mock.calls[1][0]).not.toContain("'v'");
|
||||
expect(query.mock.calls[1][1]).toEqual(["datawarehouse"]);
|
||||
expect(end).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
test("uses the typed REST table RPC and ignores non-table objects", async () => {
|
||||
const store = secretStore();
|
||||
store.put("psd-clinical", CATALOG_SECRET_IDS.apiKey, "rest-secret");
|
||||
const fetchMock = vi.fn(async () => new Response(JSON.stringify([
|
||||
{ type: "VIEW", table: "patient_view", comment: "Not a table" },
|
||||
{ type: "TABLE", table: "visits", comment: null },
|
||||
{ type: "TABLE", table: "patients", comment: "Clinical patients" },
|
||||
]), { status: 200, headers: { "content-type": "application/json" } }));
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
const postgres: CatalogPostgresAccess = {
|
||||
connect: vi.fn(async () => { throw new Error("PostgreSQL wire access must not be used"); }),
|
||||
};
|
||||
const introspector = new ConcreteCatalogTableIntrospector(postgres, store);
|
||||
|
||||
const tables = await introspector.scan(database({
|
||||
transport: "rest_api",
|
||||
baseUrl: "https://connector.internal/api/",
|
||||
restPath: "/health",
|
||||
restAuth: "x-api-key",
|
||||
}), new AbortController().signal);
|
||||
|
||||
expect(tables).toEqual([
|
||||
{ name: "patients", sourceComment: "Clinical patients" },
|
||||
{ name: "visits", sourceComment: null },
|
||||
]);
|
||||
expect(fetchMock).toHaveBeenCalledWith(
|
||||
"https://connector.internal/api/rpc/list_tables",
|
||||
expect.objectContaining({
|
||||
method: "POST",
|
||||
headers: expect.objectContaining({ "x-api-key": "rest-secret" }),
|
||||
body: JSON.stringify({ schema_name: "datawarehouse" }),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
test("fails closed when a REST table row violates the typed contract", async () => {
|
||||
const store = secretStore();
|
||||
const fetchMock = vi.fn(async () => new Response(JSON.stringify([
|
||||
{ type: "TABLE", table_name: "patients", comment: "Wrong field name" },
|
||||
]), { status: 200, headers: { "content-type": "application/json" } }));
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
const postgres: CatalogPostgresAccess = {
|
||||
connect: vi.fn(async () => { throw new Error("PostgreSQL wire access must not be used"); }),
|
||||
};
|
||||
const introspector = new ConcreteCatalogTableIntrospector(postgres, store);
|
||||
|
||||
await expect(introspector.scan(database({
|
||||
transport: "rest_api",
|
||||
baseUrl: "https://connector.internal/api",
|
||||
restPath: "/health",
|
||||
restAuth: "none",
|
||||
}), new AbortController().signal)).rejects.toThrow("REST schema response is invalid");
|
||||
});
|
||||
@@ -0,0 +1,126 @@
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { afterEach, expect, test, vi } from "vitest";
|
||||
import { buildApp } from "../src/app.js";
|
||||
import { loadConfig } from "../src/config.js";
|
||||
import { MemoryCatalogRepository } from "../src/catalog/memory-repository.js";
|
||||
import { WorkspaceSecretStore } from "../src/workspaces/secret-store.js";
|
||||
import type { WorkspaceRegistry, WorkspaceRevision } from "../src/workspaces/registry.js";
|
||||
import type { WorkspaceDescriptor } from "../src/workspaces/schema.js";
|
||||
|
||||
const roots: string[] = [];
|
||||
afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); });
|
||||
|
||||
const workspace: WorkspaceDescriptor = {
|
||||
workspace: { schema_version: 3, id: "psd-clinical", name: "Policlinico San Donato", language: "it" },
|
||||
dwh: {
|
||||
engine: "postgres", database: "warehouse", schema: "datawarehouse", port: 5432,
|
||||
supported_transports: ["postgres_direct", "rest_api"],
|
||||
},
|
||||
semantic_index: {
|
||||
vector_store: { engine: "qdrant", collection: "psd", dimensions: 1024, distance: "cosine" },
|
||||
embedding: { provider: "ollama_internal", model: "qwen3-embedding:0.6b", dimensions: 1024 },
|
||||
},
|
||||
llm_policy: { allowed: ["zai/glm-5.2"] },
|
||||
diagnostics: { dwh_rest: { method: "GET", path: "/health", auth: "bearer", response: { database: "database", schema: "schema" } } },
|
||||
};
|
||||
const revision: WorkspaceRevision = {
|
||||
id: "psd-clinical", commit: "a".repeat(40), blob: "b".repeat(40), snapshotPath: "/tmp/psd.yaml",
|
||||
};
|
||||
|
||||
async function setup() {
|
||||
const secretRoot = mkdtempSync(join(tmpdir(), "catalog-table-secret-"));
|
||||
const runtimeRoot = mkdtempSync(join(tmpdir(), "catalog-table-runtime-"));
|
||||
roots.push(secretRoot, runtimeRoot);
|
||||
const repository = new MemoryCatalogRepository();
|
||||
const database = await repository.create({
|
||||
workspaceId: "psd-clinical",
|
||||
engine: "postgres",
|
||||
databaseName: "warehouse",
|
||||
schema: "datawarehouse",
|
||||
binding: { transport: "postgres_direct", host: "db.internal", port: 5432, username: "reader" },
|
||||
});
|
||||
await repository.recordTest(database.id, database.version, {
|
||||
connectionStatus: "reachable",
|
||||
testedVersion: database.version,
|
||||
lastTestedAt: new Date().toISOString(),
|
||||
});
|
||||
const scan = vi.fn(async () => [
|
||||
{ name: "patients", sourceComment: "Clinical patients" },
|
||||
{ name: "visits", sourceComment: null },
|
||||
]);
|
||||
const registry = {
|
||||
list: vi.fn(async () => [revision]),
|
||||
listCatalog: vi.fn(async () => [{ id: "psd-clinical", name: "Policlinico San Donato", configurationState: "ready", revision }]),
|
||||
read: vi.fn(async () => ({ workspace, revision })),
|
||||
} as unknown as WorkspaceRegistry;
|
||||
const secretStore = new WorkspaceSecretStore({ root: secretRoot, runtimeRoot, installationId: "test" });
|
||||
const app = buildApp(loadConfig({ THT_HARNESS_DIR: "/missing", NODE_ENV: "test" }), {
|
||||
thtRunner: {} as never,
|
||||
workspaceRegistry: registry,
|
||||
workspaceSecretStore: secretStore,
|
||||
catalogRepository: repository,
|
||||
workspaceDiagnoser: vi.fn(),
|
||||
});
|
||||
return { app, repository, database: (await repository.get(database.id))!, scan };
|
||||
}
|
||||
|
||||
test("lists physical tables and updates only review metadata", async () => {
|
||||
const { app, repository, database, scan } = await setup();
|
||||
const synchronized = await repository.reconcileTables(database.id, database.version, await scan(), []);
|
||||
expect(synchronized).toMatchObject({ kind: "applied", createdCount: 2, deletedCount: 0 });
|
||||
expect(scan).toHaveBeenCalledOnce();
|
||||
|
||||
const tables = (await app.inject({
|
||||
method: "GET", url: `/catalog/databases/${database.id}/tables`,
|
||||
})).json();
|
||||
expect(tables.map((table: { name: string }) => table.name)).toEqual(["patients", "visits"]);
|
||||
const patients = tables[0];
|
||||
const edited = await app.inject({
|
||||
method: "PATCH",
|
||||
url: `/catalog/databases/${database.id}/tables/${patients.id}`,
|
||||
payload: { version: patients.version, description: "Curated patient registry" },
|
||||
});
|
||||
expect(edited.statusCode).toBe(200);
|
||||
expect(edited.json()).toMatchObject({
|
||||
name: "patients",
|
||||
sourceComment: "Clinical patients",
|
||||
description: "Curated patient registry",
|
||||
version: 2,
|
||||
});
|
||||
});
|
||||
|
||||
test("requires an exact deletion confirmation before applying the atomic diff", async () => {
|
||||
const { app, repository, database, scan } = await setup();
|
||||
await repository.reconcileTables(database.id, database.version, await scan(), []);
|
||||
scan.mockResolvedValue([{ name: "patients", sourceComment: "Clinical patients" }]);
|
||||
|
||||
const preview = await repository.reconcileTables(database.id, database.version, await scan(), []);
|
||||
expect(preview).toEqual({ kind: "confirmation_required", deletedNames: ["visits"] });
|
||||
expect((await app.inject({ method: "GET", url: `/catalog/databases/${database.id}/tables` })).json()).toHaveLength(2);
|
||||
|
||||
const applied = await repository.reconcileTables(database.id, database.version, await scan(), ["visits"]);
|
||||
expect(applied).toMatchObject({ kind: "applied", deletedCount: 1 });
|
||||
expect((await app.inject({ method: "GET", url: `/catalog/databases/${database.id}/tables` })).json()).toMatchObject([
|
||||
{ name: "patients" },
|
||||
]);
|
||||
});
|
||||
|
||||
test("refuses synchronization until the current binding has passed its connection test", async () => {
|
||||
const { app, repository, database } = await setup();
|
||||
await repository.update(database.id, database.version, {
|
||||
workspaceId: database.workspaceId,
|
||||
engine: database.engine,
|
||||
databaseName: database.databaseName,
|
||||
schema: database.schema,
|
||||
binding: database.binding,
|
||||
});
|
||||
const response = await app.inject({
|
||||
method: "POST",
|
||||
url: `/catalog/databases/${database.id}/sync-runs`,
|
||||
payload: { version: database.version + 1, scope: "tables", tableIds: [] },
|
||||
});
|
||||
expect(response.statusCode).toBe(409);
|
||||
expect(response.json()).toMatchObject({ code: "schema_sync_conflict" });
|
||||
});
|
||||
@@ -289,3 +289,22 @@ test("loadConfig accepts only an absolute generic model key file", () => {
|
||||
expect(() => loadConfig({ THT_MODEL_API_KEY_FILE: " /run/secrets/key" }))
|
||||
.toThrow(/model credential configuration is invalid/);
|
||||
});
|
||||
|
||||
test("loadConfig accepts a file-backed catalog role and rejects partial catalog configuration", () => {
|
||||
expect(loadConfig({
|
||||
THT_CATALOG_DB_HOST: "catalog-db",
|
||||
THT_CATALOG_DB_NAME: "thothii_catalog",
|
||||
THT_CATALOG_RUNTIME_USER: "thothii_catalog_runtime",
|
||||
THT_CATALOG_RUNTIME_PASSWORD_FILE: "/run/secrets/catalog_runtime_password",
|
||||
}).catalogDatabase).toEqual({
|
||||
host: "catalog-db",
|
||||
port: 5432,
|
||||
database: "thothii_catalog",
|
||||
user: "thothii_catalog_runtime",
|
||||
passwordFile: "/run/secrets/catalog_runtime_password",
|
||||
});
|
||||
expect(() => loadConfig({ THT_CATALOG_DB_HOST: "catalog-db" }))
|
||||
.toThrow(/catalog database configuration is invalid/);
|
||||
expect(() => loadConfig({ THT_CATALOG_DATABASE_URL: "https://catalog.invalid/db" }))
|
||||
.toThrow(/catalog database configuration is invalid/);
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user