feat: consolidate database management work

Add catalog-owned logical relationships and runtime snapshots, extend the database-management UI and validation coverage, and document the updated operational workflow.

Keep active sensitive-generation status in a tooltip and indicator, and update the layout E2E to follow the history action in its new database-scoped location.
This commit is contained in:
Codex
2026-09-01 14:46:55 +02:00
parent f586152636
commit 076c9742c5
73 changed files with 6966 additions and 610 deletions
+20
View File
@@ -63,6 +63,9 @@ import {
type DescriptionSourceSampler,
} from "./catalog/description-source-sampler.js";
import { catalogDescriptionGenerationRoutes } from "./routes/catalog-description-generation.js";
import { CatalogLogicalRelationshipService } from "./catalog/logical-relationship-service.js";
import { catalogLogicalRelationshipRoutes } from "./routes/catalog-logical-relationships.js";
import { EffectiveRelationshipSnapshotProvider } from "./catalog/effective-relationship-snapshot.js";
export interface BuildAppDeps {
thtRunner?: ThtRunner;
@@ -79,6 +82,8 @@ export interface BuildAppDeps {
catalogService?: CatalogService;
catalogPostgresAccess?: CatalogPostgresAccess;
catalogTableService?: CatalogTableService;
catalogLogicalRelationshipService?: CatalogLogicalRelationshipService;
effectiveRelationshipSnapshotProvider?: EffectiveRelationshipSnapshotProvider;
catalogSchemaIntrospector?: CatalogSchemaIntrospector;
catalogSyncWorker?: CatalogSyncWorker;
catalogOperationCoordinator?: CatalogOperationCoordinator;
@@ -200,6 +205,16 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
catalogOperationCoordinator,
);
const catalogTableService = deps?.catalogTableService ?? new CatalogTableService(catalogRepository);
const catalogLogicalRelationshipService = deps?.catalogLogicalRelationshipService
?? new CatalogLogicalRelationshipService(catalogRepository);
const effectiveRelationships = deps?.effectiveRelationshipSnapshotProvider
?? (config.catalogDatabase === undefined
? undefined
: new EffectiveRelationshipSnapshotProvider(
catalogRepository,
catalogLogicalRelationshipService,
catalogOperationCoordinator,
));
const catalogSchemaIntrospector = deps?.catalogSchemaIntrospector ?? new ConcreteCatalogSchemaIntrospector(
catalogPostgresAccess,
workspaceSecretStore,
@@ -394,6 +409,7 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
legacyWorkspaceMode: config.legacyWorkspaceMode,
workspaceRuntimeSupport,
maintenanceBarrier,
effectiveRelationships,
});
app.post("/internal/maintenance/activate", async (req, reply) => {
try {
@@ -442,6 +458,10 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
worker: catalogSyncWorker,
operations: catalogOperationCoordinator,
});
catalogLogicalRelationshipRoutes(app, {
service: catalogLogicalRelationshipService,
operations: catalogOperationCoordinator,
});
catalogDescriptionConsolidationRoutes(app, {
repository: catalogRepository,
operations: catalogOperationCoordinator,
@@ -0,0 +1,93 @@
import type { CatalogRelationship, CatalogRepository } from "./types.js";
export interface EffectiveRelationshipSnapshotReader {
list(databaseId: string): Promise<CatalogRelationship[]>;
}
export interface EffectiveRelationshipSnapshotCoordinator {
run<T>(databaseId: string, operation: () => Promise<T>): Promise<T>;
}
export class EffectiveRelationshipSnapshotStaleError extends Error {}
interface EffectiveRelationship {
sourceTable: string;
sourceColumns: string[];
targetTable: string;
targetColumns: string[];
origin: CatalogRelationship["origin"];
}
interface EffectiveRelationshipSnapshot {
schemaVersion: 1;
workspaceId: string;
relationships: EffectiveRelationship[];
}
const originRank: Record<CatalogRelationship["origin"], number> = {
physical: 0,
manual: 1,
generated: 2,
};
function endpointKey(relationship: EffectiveRelationship): string {
return [
relationship.sourceTable,
relationship.sourceColumns.join("\u0000"),
relationship.targetTable,
relationship.targetColumns.join("\u0000"),
].join("\u0001");
}
function compareRelationships(left: EffectiveRelationship, right: EffectiveRelationship): number {
return endpointKey(left).localeCompare(endpointKey(right))
|| originRank[left.origin] - originRank[right.origin];
}
/**
* Adapter from the mutable Catalog model to the immutable relationship contract consumed by the
* harness. The returned JSON is a deterministic projection, never an authored second store.
*/
export class EffectiveRelationshipSnapshotProvider {
constructor(
private readonly repository: Pick<CatalogRepository, "get" | "getByWorkspace">,
private readonly relationships: EffectiveRelationshipSnapshotReader,
private readonly operations: EffectiveRelationshipSnapshotCoordinator,
) {}
async render(workspaceId: string): Promise<string | undefined> {
const database = await this.repository.getByWorkspace(workspaceId);
if (!database) return undefined;
return await this.operations.run(database.id, async () => {
const current = await this.repository.get(database.id);
if (!current || current.schemaSyncedVersion !== current.version) {
throw new EffectiveRelationshipSnapshotStaleError(
"effective relationship snapshot requires a current full schema synchronization",
);
}
const projected = (await this.relationships.list(database.id))
.filter((relationship) => relationship.status === "active")
.map((relationship): EffectiveRelationship => ({
sourceTable: relationship.sourceTableName,
sourceColumns: relationship.columns.map((column) => column.sourceColumnName),
targetTable: relationship.targetTableName,
targetColumns: relationship.columns.map((column) => column.targetColumnName),
origin: relationship.origin,
}))
.sort(compareRelationships);
const seen = new Set<string>();
const snapshot: EffectiveRelationshipSnapshot = {
schemaVersion: 1,
workspaceId,
relationships: projected.filter((relationship) => {
const key = endpointKey(relationship);
if (seen.has(key)) return false;
seen.add(key);
return true;
}),
};
return `${JSON.stringify(snapshot, null, 2)}\n`;
});
}
}
@@ -0,0 +1,260 @@
import type {
CatalogLogicalRelationship,
CatalogLogicalRelationshipCandidate,
CatalogLogicalRelationshipContext,
CatalogLogicalRelationshipEndpoint,
CatalogRelationship,
CatalogRepository,
} from "./types.js";
export class LogicalRelationshipDatabaseNotFoundError extends Error {}
export class LogicalRelationshipDuplicateError extends Error {}
export class LogicalRelationshipNotFoundError extends Error {}
export class LogicalRelationshipReadOnlyError extends Error {}
export class LogicalRelationshipSchemaStaleError extends Error {}
export class LogicalRelationshipTargetNotUniqueError extends Error {}
export class LogicalRelationshipTypeIncompatibleError extends Error {}
export class LogicalRelationshipColumnNotFoundError extends Error {
constructor(readonly field: "sourceColumnId" | "targetColumnId") {
super(`Catalog column '${field}' was not found`);
}
}
export interface RebuildGeneratedRelationshipsResult {
added: number;
alreadyPresent: number;
excluded: number;
ambiguous: number;
}
function identifierTokens(value: string): string[] {
return value
.replace(/([a-z0-9])([A-Z])/g, "$1_$2")
.toLowerCase()
.split(/[^a-z0-9]+/)
.filter(Boolean);
}
function singularWord(value: string): string {
if (value.length > 4 && value.endsWith("ies")) return `${value.slice(0, -3)}y`;
if (value.length > 4 && /(ches|shes|xes|zes|ses)$/.test(value)) return value.slice(0, -2);
if (value.length > 3 && value.endsWith("s") && !/(ss|us)$/.test(value)) return value.slice(0, -1);
return value;
}
function tableAliases(tableName: string): string[] {
const tokens = identifierTokens(tableName);
if (tokens.length === 0) return [];
const normalized = tokens.join("_");
const singular = [...tokens];
singular[singular.length - 1] = singularWord(singular[singular.length - 1]);
return [...new Set([normalized, singular.join("_")])];
}
const GENERIC_PRIMARY_KEY_NAMES = new Set(["id", "key", "code", "pk"]);
function nameMatches(
source: CatalogLogicalRelationshipEndpoint,
target: CatalogLogicalRelationshipEndpoint,
): boolean {
const sourceName = identifierTokens(source.columnName).join("_");
const targetName = identifierTokens(target.columnName).join("_");
if (!sourceName || !targetName) return false;
const expected = new Set<string>();
if (!GENERIC_PRIMARY_KEY_NAMES.has(targetName)) expected.add(targetName);
for (const alias of tableAliases(target.tableName)) {
expected.add(`${alias}_${targetName}`);
expected.add(`${alias.replaceAll("_", "")}${targetName.replaceAll("_", "")}`);
if (targetName === "id" || targetName === "pk") expected.add(alias);
}
return expected.has(sourceName);
}
function canonicalDataType(value: string): string {
const normalized = value.trim().toLowerCase().replace(/\s+/g, " ");
const arraySuffix = normalized.endsWith("[]") ? "[]" : "";
const base = arraySuffix ? normalized.slice(0, -2) : normalized;
const withoutModifier = base.replace(/\([^)]*\)/g, "").trim();
const aliases: Record<string, string> = {
int2: "smallint",
smallserial: "smallint",
int4: "integer",
int: "integer",
serial: "integer",
int8: "bigint",
bigserial: "bigint",
decimal: "numeric",
varchar: "text",
"character varying": "text",
bool: "boolean",
"timestamp without time zone": "timestamp",
"timestamp with time zone": "timestamptz",
"time without time zone": "time",
"time with time zone": "timetz",
};
return `${aliases[withoutModifier] ?? withoutModifier}${arraySuffix}`;
}
function typesCompatible(left: string, right: string): boolean {
return canonicalDataType(left) === canonicalDataType(right);
}
function pairKey(sourceColumnId: string, targetColumnId: string): string {
return `${sourceColumnId}\u0000${targetColumnId}`;
}
function relationshipPair(relationship: CatalogLogicalRelationship): CatalogLogicalRelationshipCandidate {
return {
sourceColumnId: relationship.columns[0].sourceColumnId,
targetColumnId: relationship.columns[0].targetColumnId,
};
}
function relationshipSortKey(relationship: CatalogRelationship): string {
const sourceColumns = relationship.columns.map((column) => column.sourceColumnName).join(",");
const targetColumns = relationship.columns.map((column) => column.targetColumnName).join(",");
return [
relationship.sourceTableName,
sourceColumns,
relationship.targetTableName,
targetColumns,
relationship.origin,
].join("\u0000");
}
export class CatalogLogicalRelationshipService {
constructor(private readonly repository: CatalogRepository) {}
async list(databaseId: string): Promise<CatalogRelationship[]> {
if (!(await this.repository.get(databaseId))) throw new LogicalRelationshipDatabaseNotFoundError();
const relationships: CatalogRelationship[] = [
...await this.repository.listRelationships(databaseId),
...await this.repository.listLogicalRelationships(databaseId),
];
return relationships.sort((left, right) => relationshipSortKey(left).localeCompare(relationshipSortKey(right)));
}
async addManual(
databaseId: string,
sourceColumnId: string,
targetColumnId: string,
): Promise<CatalogLogicalRelationship> {
const context = await this.requiredContext(databaseId);
const source = context.endpoints.find((endpoint) => endpoint.columnId === sourceColumnId);
if (!source) throw new LogicalRelationshipColumnNotFoundError("sourceColumnId");
const target = context.endpoints.find((endpoint) => endpoint.columnId === targetColumnId);
if (!target) throw new LogicalRelationshipColumnNotFoundError("targetColumnId");
if (sourceColumnId === targetColumnId
|| target.primaryKeyPosition === null
|| target.tablePrimaryKeyColumnCount !== 1) {
throw new LogicalRelationshipTargetNotUniqueError();
}
if (!typesCompatible(source.dataType, target.dataType)) {
throw new LogicalRelationshipTypeIncompatibleError();
}
const key = pairKey(sourceColumnId, targetColumnId);
if (context.physicalPairs.some((pair) => pairKey(pair.sourceColumnId, pair.targetColumnId) === key)
|| context.logicalRelationships.some((relationship) => {
const pair = relationshipPair(relationship);
return pairKey(pair.sourceColumnId, pair.targetColumnId) === key;
})) throw new LogicalRelationshipDuplicateError();
const created = await this.repository.insertLogicalRelationship(
databaseId,
sourceColumnId,
targetColumnId,
false,
);
if (!created) throw new LogicalRelationshipDuplicateError();
return created;
}
async rebuildGenerated(databaseId: string): Promise<RebuildGeneratedRelationshipsResult> {
const context = await this.requiredContext(databaseId);
const targets = context.endpoints.filter((endpoint) => (
endpoint.primaryKeyPosition !== null && endpoint.tablePrimaryKeyColumnCount === 1
));
const physical = new Set(context.physicalPairs.map((pair) => pairKey(pair.sourceColumnId, pair.targetColumnId)));
const active = new Set<string>();
const excluded = new Set<string>();
for (const relationship of context.logicalRelationships) {
const pair = relationshipPair(relationship);
(relationship.status === "excluded" ? excluded : active)
.add(pairKey(pair.sourceColumnId, pair.targetColumnId));
}
const pending: CatalogLogicalRelationshipCandidate[] = [];
let alreadyPresent = 0;
let excludedCount = 0;
let ambiguous = 0;
const dimTimeTargets = targets.filter((target) => (
identifierTokens(target.tableName).join("_") === "dim_time"
));
const sources = context.endpoints.filter((endpoint) => (
endpoint.primaryKeyPosition === null || endpoint.tablePrimaryKeyColumnCount > 1
));
for (const source of sources) {
const sourceName = identifierTokens(source.columnName).join("_");
const isTimeKey = sourceName.endsWith("time_key")
&& identifierTokens(source.tableName).join("_") !== "dim_time";
const candidates = isTimeKey ? dimTimeTargets : targets;
const matches = candidates.filter((target) => (
target.columnId !== source.columnId
&& typesCompatible(source.dataType, target.dataType)
&& (isTimeKey || nameMatches(source, target))
));
if (matches.length > 1) {
ambiguous += 1;
continue;
}
if (matches.length === 0) continue;
const candidate = { sourceColumnId: source.columnId, targetColumnId: matches[0].columnId };
const key = pairKey(candidate.sourceColumnId, candidate.targetColumnId);
if (physical.has(key) || active.has(key)) {
alreadyPresent += 1;
} else if (excluded.has(key)) {
excludedCount += 1;
} else {
pending.push(candidate);
}
}
const added = await this.repository.insertGeneratedLogicalRelationships(databaseId, pending);
alreadyPresent += pending.length - added;
return { added, alreadyPresent, excluded: excludedCount, ambiguous };
}
async setStatus(
databaseId: string,
relationshipId: string,
status: CatalogLogicalRelationship["status"],
): Promise<CatalogLogicalRelationship> {
if (!(await this.repository.get(databaseId))) throw new LogicalRelationshipDatabaseNotFoundError();
const updated = await this.repository.setLogicalRelationshipStatus(databaseId, relationshipId, status);
if (updated) return updated;
await this.assertNotPhysical(databaseId, relationshipId);
throw new LogicalRelationshipNotFoundError();
}
async deletePermanently(databaseId: string, relationshipId: string): Promise<void> {
if (!(await this.repository.get(databaseId))) throw new LogicalRelationshipDatabaseNotFoundError();
if (await this.repository.deleteLogicalRelationship(databaseId, relationshipId)) return;
await this.assertNotPhysical(databaseId, relationshipId);
throw new LogicalRelationshipNotFoundError();
}
private async requiredContext(databaseId: string): Promise<CatalogLogicalRelationshipContext> {
const database = await this.repository.get(databaseId);
if (!database) throw new LogicalRelationshipDatabaseNotFoundError();
if (database.schemaSyncedVersion !== database.version) {
throw new LogicalRelationshipSchemaStaleError();
}
const context = await this.repository.getLogicalRelationshipContext(databaseId);
if (!context) throw new LogicalRelationshipDatabaseNotFoundError();
return context;
}
private async assertNotPhysical(databaseId: string, relationshipId: string): Promise<void> {
if ((await this.repository.listRelationships(databaseId)).some((relationship) => relationship.id === relationshipId)) {
throw new LogicalRelationshipReadOnlyError();
}
}
}
+150 -3
View File
@@ -14,7 +14,10 @@ import {
type CatalogDatabaseMetadataDeleteTarget,
type CatalogMetadataDeleteCounts,
type CatalogMetrics,
type CatalogRelationship,
type CatalogLogicalRelationship,
type CatalogLogicalRelationshipCandidate,
type CatalogLogicalRelationshipContext,
type CatalogPhysicalRelationship,
type CatalogSchemaDiff,
type CatalogSyncCounts,
type CatalogSyncEvent,
@@ -49,7 +52,8 @@ export class MemoryCatalogRepository implements CatalogRepository {
private readonly records = new Map<string, WorkspaceDatabase>();
private readonly tables = new Map<string, CatalogTable>();
private readonly columns = new Map<string, CatalogColumn>();
private readonly relationships = new Map<string, CatalogRelationship>();
private readonly relationships = new Map<string, CatalogPhysicalRelationship>();
private readonly logicalRelationships = new Map<string, CatalogLogicalRelationship>();
private readonly descriptionGenerationRuns = new Map<string, DescriptionGenerationRun>();
private readonly descriptionGenerationEvents = new Map<string, DescriptionGenerationEvent[]>();
private readonly sensitiveDataSuggestionRuns = new Map<string, SensitiveDataSuggestionRun>();
@@ -171,6 +175,9 @@ export class MemoryCatalogRepository implements CatalogRepository {
for (const [relationshipId, relationship] of this.relationships) {
if (relationship.databaseId === id) this.relationships.delete(relationshipId);
}
for (const [relationshipId, relationship] of this.logicalRelationships) {
if (relationship.databaseId === id) this.logicalRelationships.delete(relationshipId);
}
for (const [runId, run] of this.descriptionGenerationRuns) {
if (run.databaseId !== id) continue;
this.descriptionGenerationRuns.delete(runId);
@@ -527,12 +534,138 @@ export class MemoryCatalogRepository implements CatalogRepository {
.map((event) => structuredClone(event));
}
async listRelationships(databaseId: string): Promise<CatalogRelationship[]> {
async listRelationships(databaseId: string): Promise<CatalogPhysicalRelationship[]> {
return [...this.relationships.values()].filter((relationship) => relationship.databaseId === databaseId)
.sort((a, b) => `${a.sourceTableName}.${a.constraintName}`.localeCompare(`${b.sourceTableName}.${b.constraintName}`))
.map((relationship) => structuredClone(relationship));
}
async listLogicalRelationships(databaseId: string): Promise<CatalogLogicalRelationship[]> {
return [...this.logicalRelationships.values()]
.filter((relationship) => relationship.databaseId === databaseId)
.sort((a, b) => {
const left = `${a.sourceTableName}.${a.columns[0].sourceColumnName}.${a.targetTableName}.${a.columns[0].targetColumnName}`;
const right = `${b.sourceTableName}.${b.columns[0].sourceColumnName}.${b.targetTableName}.${b.columns[0].targetColumnName}`;
return left.localeCompare(right);
})
.map((relationship) => structuredClone(relationship));
}
async getLogicalRelationshipContext(
databaseId: string,
): Promise<CatalogLogicalRelationshipContext | undefined> {
if (!this.records.has(databaseId)) return undefined;
const tables = [...this.tables.values()].filter((table) => table.databaseId === databaseId);
const tableById = new Map(tables.map((table) => [table.id, table]));
const columns = [...this.columns.values()].filter((column) => tableById.has(column.tableId));
const primaryKeyCounts = new Map<string, number>();
for (const column of columns) {
if (column.primaryKeyPosition !== null) {
primaryKeyCounts.set(column.tableId, (primaryKeyCounts.get(column.tableId) ?? 0) + 1);
}
}
return {
endpoints: columns.map((column) => ({
columnId: column.id,
columnName: column.name,
tableId: column.tableId,
tableName: tableById.get(column.tableId)!.name,
dataType: column.dataType,
primaryKeyPosition: column.primaryKeyPosition,
tablePrimaryKeyColumnCount: primaryKeyCounts.get(column.tableId) ?? 0,
})),
physicalPairs: [...this.relationships.values()]
.filter((relationship) => relationship.databaseId === databaseId)
.flatMap((relationship) => relationship.columns.map((column) => ({
sourceColumnId: column.sourceColumnId,
targetColumnId: column.targetColumnId,
}))),
logicalRelationships: await this.listLogicalRelationships(databaseId),
};
}
async insertLogicalRelationship(
databaseId: string,
sourceColumnId: string,
targetColumnId: string,
generated: boolean,
): Promise<CatalogLogicalRelationship | undefined> {
if ([...this.logicalRelationships.values()].some((relationship) => (
relationship.databaseId === databaseId
&& relationship.columns[0].sourceColumnId === sourceColumnId
&& relationship.columns[0].targetColumnId === targetColumnId
))) return undefined;
const sourceColumn = this.columns.get(sourceColumnId);
const targetColumn = this.columns.get(targetColumnId);
const sourceTable = sourceColumn ? this.tables.get(sourceColumn.tableId) : undefined;
const targetTable = targetColumn ? this.tables.get(targetColumn.tableId) : undefined;
if (!sourceColumn || !targetColumn || !sourceTable || !targetTable
|| sourceTable.databaseId !== databaseId || targetTable.databaseId !== databaseId
|| sourceColumnId === targetColumnId) return undefined;
const now = new Date().toISOString();
const relationship: CatalogLogicalRelationship = {
id: randomUUID(),
databaseId,
constraintName: null,
sourceTableId: sourceTable.id,
sourceTableName: sourceTable.name,
targetTableId: targetTable.id,
targetTableName: targetTable.name,
updateRule: null,
deleteRule: null,
deferrable: false,
initiallyDeferred: false,
columns: [{
position: 1,
sourceColumnId,
sourceColumnName: sourceColumn.name,
targetColumnId,
targetColumnName: targetColumn.name,
}],
lastSyncedDatabaseVersion: null,
lastSyncedAt: null,
createdAt: now,
updatedAt: now,
origin: generated ? "generated" : "manual",
status: "active",
};
this.logicalRelationships.set(relationship.id, relationship);
return structuredClone(relationship);
}
async insertGeneratedLogicalRelationships(
databaseId: string,
candidates: readonly CatalogLogicalRelationshipCandidate[],
): Promise<number> {
let added = 0;
for (const candidate of candidates) {
if (await this.insertLogicalRelationship(
databaseId,
candidate.sourceColumnId,
candidate.targetColumnId,
true,
)) added += 1;
}
return added;
}
async setLogicalRelationshipStatus(
databaseId: string,
relationshipId: string,
status: CatalogLogicalRelationship["status"],
): Promise<CatalogLogicalRelationship | undefined> {
const current = this.logicalRelationships.get(relationshipId);
if (!current || current.databaseId !== databaseId) return undefined;
const updated = { ...current, status, updatedAt: new Date().toISOString() };
this.logicalRelationships.set(relationshipId, updated);
return structuredClone(updated);
}
async deleteLogicalRelationship(databaseId: string, relationshipId: string): Promise<boolean> {
const current = this.logicalRelationships.get(relationshipId);
return Boolean(current?.databaseId === databaseId && this.logicalRelationships.delete(relationshipId));
}
async deleteDatabaseMetadata(
databaseIds: readonly string[],
target: CatalogDatabaseMetadataDeleteTarget,
@@ -574,6 +707,12 @@ export class MemoryCatalogRepository implements CatalogRepository {
const columns = [...this.columns.values()].filter((column) => selected.has(column.tableId));
const deletedColumnIds = new Set(columns.map((column) => column.id));
for (const column of columns) this.columns.delete(column.id);
for (const relationship of [...this.logicalRelationships.values()]) {
const pair = relationship.columns[0];
if (deletedColumnIds.has(pair.sourceColumnId) || deletedColumnIds.has(pair.targetColumnId)) {
this.logicalRelationships.delete(relationship.id);
}
}
for (const [relationshipId, relationship] of this.relationships) {
if (relationship.databaseId !== databaseId) continue;
this.relationships.set(relationshipId, {
@@ -857,6 +996,8 @@ export class MemoryCatalogRepository implements CatalogRepository {
lastSyncedAt: now,
createdAt: current?.createdAt ?? now,
updatedAt: comparable === nextComparable ? (current?.updatedAt ?? now) : now,
origin: "physical",
status: "active",
});
if (!current) created += 1;
else if (comparable !== nextComparable) updated += 1;
@@ -1073,6 +1214,12 @@ export class MemoryCatalogRepository implements CatalogRepository {
this.relationships.delete(relationship.id);
}
}
for (const relationship of [...this.logicalRelationships.values()]) {
const pair = relationship.columns[0];
if (pair.sourceColumnId === columnId || pair.targetColumnId === columnId) {
this.logicalRelationships.delete(relationship.id);
}
}
}
private markCatalogIncomplete(databaseIds: readonly string[]): void {
+2
View File
@@ -10,6 +10,7 @@ import * as catalogRuntimeSequencePrivilegesMigration from "./migrations/004_cat
import * as descriptionGenerationRunsMigration from "./migrations/005_description_generation_runs.js";
import * as sensitiveDataFlagMigration from "./migrations/006_sensitive_data_flag.js";
import * as sensitiveDataSuggestionRunsMigration from "./migrations/007_sensitive_data_suggestion_runs.js";
import * as catalogLogicalRelationshipsMigration from "./migrations/008_catalog_logical_relationships.js";
const connectionString = process.env.THT_CATALOG_MIGRATOR_DATABASE_URL;
const host = process.env.THT_CATALOG_DB_HOST;
@@ -42,6 +43,7 @@ const provider: MigrationProvider = {
"005_description_generation_runs": descriptionGenerationRunsMigration,
"006_sensitive_data_flag": sensitiveDataFlagMigration,
"007_sensitive_data_suggestion_runs": sensitiveDataSuggestionRunsMigration,
"008_catalog_logical_relationships": catalogLogicalRelationshipsMigration,
};
},
};
@@ -0,0 +1,35 @@
import { type Kysely, sql } from "kysely";
import type { CatalogDatabase } from "../repository.js";
export async function up(db: Kysely<CatalogDatabase>): Promise<void> {
await db.schema.createTable("catalog_logical_relationships")
.addColumn("id", "uuid", (column) => column.primaryKey())
.addColumn("database_id", "uuid", (column) => column.notNull()
.references("workspace_databases.id").onDelete("cascade"))
.addColumn("source_column_id", "uuid", (column) => column.notNull()
.references("catalog_columns.id").onDelete("cascade"))
.addColumn("target_column_id", "uuid", (column) => column.notNull()
.references("catalog_columns.id").onDelete("cascade"))
.addColumn("generated", "boolean", (column) => column.notNull().defaultTo(false))
.addColumn("deleted_at", "timestamptz")
.addColumn("created_at", "timestamptz", (column) => column.notNull().defaultTo(sql`now()`))
.addColumn("updated_at", "timestamptz", (column) => column.notNull().defaultTo(sql`now()`))
.addUniqueConstraint(
"catalog_logical_relationships_endpoint_key",
["database_id", "source_column_id", "target_column_id"],
)
.addCheckConstraint(
"catalog_logical_relationships_distinct_columns_check",
sql`source_column_id <> target_column_id`,
)
.execute();
await db.schema.createIndex("catalog_logical_relationships_database_deleted_idx")
.on("catalog_logical_relationships")
.columns(["database_id", "deleted_at"])
.execute();
}
export async function down(db: Kysely<CatalogDatabase>): Promise<void> {
await db.schema.dropTable("catalog_logical_relationships").execute();
}
+176 -4
View File
@@ -23,7 +23,10 @@ import {
type CatalogDatabaseMetadataDeleteTarget,
type CatalogMetadataDeleteCounts,
type CatalogMetrics,
type CatalogRelationship,
type CatalogLogicalRelationship,
type CatalogLogicalRelationshipCandidate,
type CatalogLogicalRelationshipContext,
type CatalogPhysicalRelationship,
type CatalogSchemaDiff,
type CatalogSyncCounts,
type CatalogSyncEvent,
@@ -145,6 +148,17 @@ interface CatalogRelationshipColumnTable {
targetColumnId: string;
}
interface CatalogLogicalRelationshipTable {
id: string;
databaseId: string;
sourceColumnId: string;
targetColumnId: string;
generated: Generated<boolean>;
deletedAt: Timestamp | null;
createdAt: Timestamp;
updatedAt: Timestamp;
}
interface DescriptionGenerationRunTable {
id: string;
databaseId: string;
@@ -241,6 +255,7 @@ export interface CatalogDatabase {
catalogColumns: CatalogColumnTable;
catalogRelationships: CatalogRelationshipTable;
catalogRelationshipColumns: CatalogRelationshipColumnTable;
catalogLogicalRelationships: CatalogLogicalRelationshipTable;
descriptionGenerationRuns: DescriptionGenerationRunTable;
descriptionGenerationEvents: DescriptionGenerationEventTable;
sensitiveDataSuggestionRuns: SensitiveDataSuggestionRunTable;
@@ -1022,7 +1037,7 @@ export class KyselyCatalogRepository implements CatalogRepository {
return rows.map(serializeSensitiveDataSuggestionEvent);
}
async listRelationships(databaseId: string): Promise<CatalogRelationship[]> {
async listRelationships(databaseId: string): Promise<CatalogPhysicalRelationship[]> {
const rows = await this.db.selectFrom("catalogRelationships as relationship")
.innerJoin("catalogTables as sourceTable", "sourceTable.id", "relationship.sourceTableId")
.innerJoin("catalogTables as targetTable", "targetTable.id", "relationship.targetTableId")
@@ -1049,7 +1064,7 @@ export class KyselyCatalogRepository implements CatalogRepository {
.where("pair.relationshipId", "in", rows.map((row) => row.id))
.orderBy("pair.relationshipId").orderBy("pair.position")
.execute();
const byRelationship = new Map<string, CatalogRelationship["columns"]>();
const byRelationship = new Map<string, CatalogPhysicalRelationship["columns"]>();
for (const pair of pairs) {
const items = byRelationship.get(pair.relationshipId) ?? [];
items.push(pair);
@@ -1061,9 +1076,160 @@ export class KyselyCatalogRepository implements CatalogRepository {
lastSyncedAt: row.lastSyncedAt === null ? null : new Date(row.lastSyncedAt).toISOString(),
createdAt: new Date(row.createdAt).toISOString(),
updatedAt: new Date(row.updatedAt).toISOString(),
origin: "physical" as const,
status: "active" as const,
}));
}
async listLogicalRelationships(databaseId: string): Promise<CatalogLogicalRelationship[]> {
const rows = await this.db.selectFrom("catalogLogicalRelationships as relationship")
.innerJoin("catalogColumns as sourceColumn", "sourceColumn.id", "relationship.sourceColumnId")
.innerJoin("catalogTables as sourceTable", "sourceTable.id", "sourceColumn.tableId")
.innerJoin("catalogColumns as targetColumn", "targetColumn.id", "relationship.targetColumnId")
.innerJoin("catalogTables as targetTable", "targetTable.id", "targetColumn.tableId")
.select([
"relationship.id", "relationship.databaseId", "relationship.generated",
"relationship.deletedAt", "relationship.createdAt", "relationship.updatedAt",
"sourceTable.id as sourceTableId", "sourceTable.name as sourceTableName",
"sourceColumn.id as sourceColumnId", "sourceColumn.name as sourceColumnName",
"targetTable.id as targetTableId", "targetTable.name as targetTableName",
"targetColumn.id as targetColumnId", "targetColumn.name as targetColumnName",
])
.where("relationship.databaseId", "=", databaseId)
.orderBy("sourceTable.name")
.orderBy("sourceColumn.name")
.orderBy("targetTable.name")
.orderBy("targetColumn.name")
.execute();
return rows.map((row) => ({
id: row.id,
databaseId: row.databaseId,
constraintName: null,
sourceTableId: row.sourceTableId,
sourceTableName: row.sourceTableName,
targetTableId: row.targetTableId,
targetTableName: row.targetTableName,
updateRule: null,
deleteRule: null,
deferrable: false,
initiallyDeferred: false,
columns: [{
position: 1,
sourceColumnId: row.sourceColumnId,
sourceColumnName: row.sourceColumnName,
targetColumnId: row.targetColumnId,
targetColumnName: row.targetColumnName,
}],
lastSyncedDatabaseVersion: null,
lastSyncedAt: null,
createdAt: new Date(row.createdAt).toISOString(),
updatedAt: new Date(row.updatedAt).toISOString(),
origin: row.generated ? "generated" : "manual",
status: row.deletedAt === null ? "active" : "excluded",
}));
}
async getLogicalRelationshipContext(
databaseId: string,
): Promise<CatalogLogicalRelationshipContext | undefined> {
if (!(await selectOne(this.db, databaseId))) return undefined;
const columns = await this.db.selectFrom("catalogColumns as column")
.innerJoin("catalogTables as table", "table.id", "column.tableId")
.select([
"column.id as columnId", "column.name as columnName", "column.dataType",
"column.primaryKeyPosition", "table.id as tableId", "table.name as tableName",
])
.where("table.databaseId", "=", databaseId)
.orderBy("table.name")
.orderBy("column.ordinalPosition")
.execute();
const primaryKeyCounts = new Map<string, number>();
for (const column of columns) {
if (column.primaryKeyPosition !== null) {
primaryKeyCounts.set(column.tableId, (primaryKeyCounts.get(column.tableId) ?? 0) + 1);
}
}
const physicalPairs = await this.db.selectFrom("catalogRelationshipColumns as pair")
.innerJoin("catalogRelationships as relationship", "relationship.id", "pair.relationshipId")
.select(["pair.sourceColumnId", "pair.targetColumnId"])
.where("relationship.databaseId", "=", databaseId)
.execute();
return {
endpoints: columns.map((column) => ({
...column,
tablePrimaryKeyColumnCount: primaryKeyCounts.get(column.tableId) ?? 0,
})),
physicalPairs,
logicalRelationships: await this.listLogicalRelationships(databaseId),
};
}
async insertLogicalRelationship(
databaseId: string,
sourceColumnId: string,
targetColumnId: string,
generated: boolean,
): Promise<CatalogLogicalRelationship | undefined> {
const id = randomUUID();
const inserted = await this.db.insertInto("catalogLogicalRelationships").values({
id, databaseId, sourceColumnId, targetColumnId, generated,
}).onConflict((conflict) => conflict
.columns(["databaseId", "sourceColumnId", "targetColumnId"])
.doNothing())
.returning("id")
.executeTakeFirst();
if (!inserted) return undefined;
return (await this.listLogicalRelationships(databaseId)).find((item) => item.id === id);
}
async insertGeneratedLogicalRelationships(
databaseId: string,
candidates: readonly CatalogLogicalRelationshipCandidate[],
): Promise<number> {
if (candidates.length === 0) return 0;
return await this.db.transaction().execute(async (trx) => {
let added = 0;
for (const candidate of candidates) {
const inserted = await trx.insertInto("catalogLogicalRelationships").values({
id: randomUUID(),
databaseId,
sourceColumnId: candidate.sourceColumnId,
targetColumnId: candidate.targetColumnId,
generated: true,
}).onConflict((conflict) => conflict
.columns(["databaseId", "sourceColumnId", "targetColumnId"])
.doNothing())
.returning("id")
.executeTakeFirst();
if (inserted) added += 1;
}
return added;
});
}
async setLogicalRelationshipStatus(
databaseId: string,
relationshipId: string,
status: CatalogLogicalRelationship["status"],
): Promise<CatalogLogicalRelationship | undefined> {
const updated = await this.db.updateTable("catalogLogicalRelationships")
.set({ deletedAt: status === "excluded" ? sql`now()` : null, updatedAt: sql`now()` })
.where("databaseId", "=", databaseId)
.where("id", "=", relationshipId)
.returning("id")
.executeTakeFirst();
if (!updated) return undefined;
return (await this.listLogicalRelationships(databaseId)).find((item) => item.id === relationshipId);
}
async deleteLogicalRelationship(databaseId: string, relationshipId: string): Promise<boolean> {
const result = await this.db.deleteFrom("catalogLogicalRelationships")
.where("databaseId", "=", databaseId)
.where("id", "=", relationshipId)
.executeTakeFirst();
return result.numDeletedRows > 0n;
}
async deleteDatabaseMetadata(
databaseIds: readonly string[],
target: CatalogDatabaseMetadataDeleteTarget,
@@ -1621,7 +1787,13 @@ export class UnavailableCatalogRepository implements CatalogRepository {
async updateSensitiveDataSuggestionRun(): Promise<SensitiveDataSuggestionRun | undefined> { return this.fail(); }
async appendSensitiveDataSuggestionEvent(): Promise<SensitiveDataSuggestionEvent> { return this.fail(); }
async listSensitiveDataSuggestionEvents(): Promise<SensitiveDataSuggestionEvent[]> { return this.fail(); }
async listRelationships(): Promise<CatalogRelationship[]> { return this.fail(); }
async listRelationships(): Promise<CatalogPhysicalRelationship[]> { return this.fail(); }
async listLogicalRelationships(): Promise<CatalogLogicalRelationship[]> { return this.fail(); }
async getLogicalRelationshipContext(): Promise<CatalogLogicalRelationshipContext | undefined> { return this.fail(); }
async insertLogicalRelationship(): Promise<CatalogLogicalRelationship | undefined> { return this.fail(); }
async insertGeneratedLogicalRelationships(): Promise<number> { return this.fail(); }
async setLogicalRelationshipStatus(): Promise<CatalogLogicalRelationship | undefined> { return this.fail(); }
async deleteLogicalRelationship(): Promise<boolean> { return this.fail(); }
async deleteDatabaseMetadata(): Promise<CatalogMetadataDeleteCounts | undefined> { return this.fail(); }
async deleteTableMetadata(): Promise<CatalogMetadataDeleteCounts | undefined> { return this.fail(); }
async planSchemaSync(): Promise<CatalogSchemaDiff> { return this.fail(); }
+65 -2
View File
@@ -2,6 +2,7 @@ import { buildInstallationContract } from "../workspaces/contracts.js";
import { resolveBinding } from "../workspaces/bindings.js";
import type { WorkspaceRegistry } from "../workspaces/registry.js";
import type { WorkspaceDescriptor } from "../workspaces/schema.js";
import { discoverWorkspaceSecretRequirements } from "../workspaces/secret-requirements.js";
import type { WorkspaceSecretStore } from "../workspaces/secret-store.js";
import { createConcreteDiagnosticAdapters } from "../workspaces/diagnostics.js";
import { CatalogOperationCoordinator } from "./operation-coordinator.js";
@@ -25,6 +26,21 @@ export interface CatalogListItem extends Omit<WorkspaceDatabase, "id"> {
workspaceName: string;
workspaceDescription?: string;
workspaceAvailable: boolean;
workspaceRevision: { commit: string; blob: string } | null;
workspaceEvidence: {
sourceType: "filesystem" | "http" | "s3" | null;
state:
| "not_declared"
| "materialized_current_revision"
| "configuration_required"
| "configured_unverified"
| "workspace_unavailable";
};
runtimeBinding: {
transport: DatabaseBinding["transport"];
configurationState: "ready" | "configuration_required";
sessionTransportSupported: boolean;
} | null;
configured: boolean;
secrets: Record<CatalogSecretName, boolean>;
}
@@ -68,6 +84,43 @@ function secretState(store: WorkspaceSecretStore, workspaceId: string): Record<C
))) as Record<CatalogSecretName, boolean>;
}
function workspaceRuntimeState(
store: WorkspaceSecretStore,
workspace: WorkspaceDescriptor,
secretRoots: readonly string[],
): NonNullable<CatalogListItem["runtimeBinding"]> {
const requirements = discoverWorkspaceSecretRequirements(workspace, process.env);
const secretVariables = new Set(requirements.map(({ variable }) => variable));
const effective = resolveBinding(workspace, "DWH", process.env, secretRoots);
const configurationRequired = requirements.some(({ id, required }) => (
required && !store.has(workspace.workspace.id, id)
)) || effective.missing.some((variable) => !secretVariables.has(variable));
return {
transport: effective.transport,
configurationState: configurationRequired ? "configuration_required" : "ready",
sessionTransportSupported: effective.transport !== "ssh_tunnel",
};
}
function workspaceEvidenceState(
store: WorkspaceSecretStore,
workspace: WorkspaceDescriptor,
): CatalogListItem["workspaceEvidence"] {
const source = workspace.evidence?.source;
if (!source) return { sourceType: null, state: "not_declared" };
if (source.type === "filesystem") {
return { sourceType: source.type, state: "materialized_current_revision" };
}
const configurationRequired = discoverWorkspaceSecretRequirements(workspace, process.env)
.some(({ connector, id, required }) => (
connector === "evidence" && required && !store.has(workspace.workspace.id, id)
));
return {
sourceType: source.type,
state: configurationRequired ? "configuration_required" : "configured_unverified",
};
}
export class CatalogService {
private readonly adapters = createConcreteDiagnosticAdapters();
@@ -91,7 +144,8 @@ export class CatalogService {
const byWorkspace = new Map(configured.map((database) => [database.workspaceId, database]));
const active = await Promise.all(workspaces.map(async (entry) => {
const database = byWorkspace.get(entry.id);
const { workspace } = await this.registry.read(entry.id);
const { workspace } = await this.registry.readPinned(entry.id, entry.revision.commit);
const runtimeDatabaseBinding = yamlBinding(workspace, this.secretRoots);
const base = database ?? {
workspaceId: entry.id,
engine: "postgres" as const,
@@ -100,7 +154,7 @@ export class CatalogService {
version: 0,
createdAt: "",
updatedAt: "",
binding: yamlBinding(workspace, this.secretRoots),
binding: runtimeDatabaseBinding,
connectionStatus: "untested" as const,
};
return {
@@ -108,6 +162,12 @@ export class CatalogService {
workspaceName: entry.name,
workspaceDescription: entry.description,
workspaceAvailable: true,
workspaceRevision: {
commit: entry.revision.commit,
blob: entry.revision.blob,
},
workspaceEvidence: workspaceEvidenceState(this.secretStore, workspace),
runtimeBinding: workspaceRuntimeState(this.secretStore, workspace, this.secretRoots),
configured: database !== undefined,
secrets: secretState(this.secretStore, entry.id),
};
@@ -120,6 +180,9 @@ export class CatalogService {
workspaceName: database.workspaceId,
workspaceDescription: "Workspace is no longer present in the repository catalog.",
workspaceAvailable: false,
workspaceRevision: null,
workspaceEvidence: { sourceType: null, state: "workspace_unavailable" },
runtimeBinding: null,
configured: true,
secrets: secretState(this.secretStore, database.workspaceId),
}));
+66 -2
View File
@@ -128,7 +128,7 @@ export interface CatalogRelationshipColumn {
targetColumnName: string;
}
export interface CatalogRelationship {
export interface CatalogPhysicalRelationship {
id: string;
databaseId: string;
constraintName: string;
@@ -145,6 +145,52 @@ export interface CatalogRelationship {
lastSyncedAt: string | null;
createdAt: string;
updatedAt: string;
origin: "physical";
status: "active";
}
export interface CatalogLogicalRelationship {
id: string;
databaseId: string;
constraintName: null;
sourceTableId: string;
sourceTableName: string;
targetTableId: string;
targetTableName: string;
updateRule: null;
deleteRule: null;
deferrable: false;
initiallyDeferred: false;
columns: [CatalogRelationshipColumn];
lastSyncedDatabaseVersion: null;
lastSyncedAt: null;
createdAt: string;
updatedAt: string;
origin: "generated" | "manual";
status: "active" | "excluded";
}
export type CatalogRelationship = CatalogPhysicalRelationship | CatalogLogicalRelationship;
export interface CatalogLogicalRelationshipEndpoint {
columnId: string;
columnName: string;
tableId: string;
tableName: string;
dataType: string;
primaryKeyPosition: number | null;
tablePrimaryKeyColumnCount: number;
}
export interface CatalogLogicalRelationshipContext {
endpoints: CatalogLogicalRelationshipEndpoint[];
physicalPairs: Array<{ sourceColumnId: string; targetColumnId: string }>;
logicalRelationships: CatalogLogicalRelationship[];
}
export interface CatalogLogicalRelationshipCandidate {
sourceColumnId: string;
targetColumnId: string;
}
export type CatalogDatabaseMetadataDeleteTarget = "tables" | "relationships";
@@ -456,7 +502,25 @@ export interface CatalogRepository {
runId: string,
afterSequence?: number,
): Promise<SensitiveDataSuggestionEvent[]>;
listRelationships(databaseId: string): Promise<CatalogRelationship[]>;
listRelationships(databaseId: string): Promise<CatalogPhysicalRelationship[]>;
listLogicalRelationships(databaseId: string): Promise<CatalogLogicalRelationship[]>;
getLogicalRelationshipContext(databaseId: string): Promise<CatalogLogicalRelationshipContext | undefined>;
insertLogicalRelationship(
databaseId: string,
sourceColumnId: string,
targetColumnId: string,
generated: boolean,
): Promise<CatalogLogicalRelationship | undefined>;
insertGeneratedLogicalRelationships(
databaseId: string,
candidates: readonly CatalogLogicalRelationshipCandidate[],
): Promise<number>;
setLogicalRelationshipStatus(
databaseId: string,
relationshipId: string,
status: CatalogLogicalRelationship["status"],
): Promise<CatalogLogicalRelationship | undefined>;
deleteLogicalRelationship(databaseId: string, relationshipId: string): Promise<boolean>;
deleteDatabaseMetadata(
databaseIds: readonly string[],
target: CatalogDatabaseMetadataDeleteTarget,
@@ -0,0 +1,154 @@
import type { FastifyInstance, FastifyReply, FastifyRequest } from "fastify";
import { z } from "zod";
import { isPrincipalContext, requirePermission } from "../auth/authorization.js";
import {
CatalogLogicalRelationshipService,
LogicalRelationshipColumnNotFoundError,
LogicalRelationshipDatabaseNotFoundError,
LogicalRelationshipDuplicateError,
LogicalRelationshipNotFoundError,
LogicalRelationshipReadOnlyError,
LogicalRelationshipSchemaStaleError,
LogicalRelationshipTargetNotUniqueError,
LogicalRelationshipTypeIncompatibleError,
} from "../catalog/logical-relationship-service.js";
import type { CatalogOperationCoordinator } from "../catalog/operation-coordinator.js";
import { CatalogOperationInProgressError, CatalogUnavailableError } from "../catalog/types.js";
const idSchema = z.uuid();
const createSchema = z.object({
sourceColumnId: idSchema,
targetColumnId: idSchema,
}).strict();
const statusSchema = z.object({ status: z.enum(["active", "excluded"]) }).strict();
const emptySchema = z.object({}).strict();
function manage(request: FastifyRequest, reply: FastifyReply) {
return isPrincipalContext(requirePermission(request, reply, "database.manage"));
}
function safeError(reply: FastifyReply, error: unknown) {
if (error instanceof CatalogUnavailableError) {
return reply.code(503).send({ code: "catalog_unavailable", message: "Database catalog is unavailable." });
}
if (error instanceof CatalogOperationInProgressError) {
return reply.code(409).send({
code: "database_operation_in_progress",
message: "A database operation is already in progress.",
});
}
if (error instanceof LogicalRelationshipDatabaseNotFoundError) {
return reply.code(404).send({ code: "database_not_found", message: "Database configuration was not found." });
}
if (error instanceof LogicalRelationshipColumnNotFoundError) {
return reply.code(404).send({
code: "column_not_found",
message: "Catalog column was not found.",
field: error.field,
});
}
if (error instanceof LogicalRelationshipNotFoundError) {
return reply.code(404).send({ code: "relationship_not_found", message: "Relationship was not found." });
}
if (error instanceof LogicalRelationshipDuplicateError) {
return reply.code(409).send({ code: "relationship_duplicate", message: "Relationship already exists." });
}
if (error instanceof LogicalRelationshipReadOnlyError) {
return reply.code(409).send({ code: "relationship_read_only", message: "Physical relationships are read-only." });
}
if (error instanceof LogicalRelationshipSchemaStaleError) {
return reply.code(409).send({
code: "relationship_schema_stale",
message: "Synchronize the current database schema before managing logical relationships.",
});
}
if (error instanceof LogicalRelationshipTargetNotUniqueError) {
return reply.code(422).send({
code: "relationship_target_not_unique",
message: "Target column must be the only primary-key column of its table.",
field: "targetColumnId",
});
}
if (error instanceof LogicalRelationshipTypeIncompatibleError) {
return reply.code(422).send({
code: "relationship_type_incompatible",
message: "Source and target column types are incompatible.",
field: "targetColumnId",
});
}
if (error instanceof z.ZodError) {
return reply.code(400).send({
code: "relationship_request_invalid",
message: "Relationship request is invalid.",
});
}
return reply.code(500).send({
code: "relationship_operation_failed",
message: "Relationship operation failed.",
});
}
export function catalogLogicalRelationshipRoutes(
app: FastifyInstance,
deps: {
service: CatalogLogicalRelationshipService;
operations: CatalogOperationCoordinator;
},
): void {
app.get("/catalog/databases/:databaseId/relationships", async (request, reply) => {
if (!manage(request, reply)) return reply;
try {
const databaseId = idSchema.parse((request.params as { databaseId?: unknown }).databaseId);
return await deps.service.list(databaseId);
} catch (error) { return safeError(reply, error); }
});
app.post("/catalog/databases/:databaseId/relationships", async (request, reply) => {
if (!manage(request, reply)) return reply;
try {
const databaseId = idSchema.parse((request.params as { databaseId?: unknown }).databaseId);
const input = createSchema.parse(request.body);
const relationship = await deps.operations.run(databaseId, async () => (
await deps.service.addManual(databaseId, input.sourceColumnId, input.targetColumnId)
));
return reply.code(201).send(relationship);
} catch (error) { return safeError(reply, error); }
});
app.post("/catalog/databases/:databaseId/relationships/rebuild-generated", async (request, reply) => {
if (!manage(request, reply)) return reply;
try {
const databaseId = idSchema.parse((request.params as { databaseId?: unknown }).databaseId);
emptySchema.parse(request.body ?? {});
return await deps.operations.run(databaseId, async () => (
await deps.service.rebuildGenerated(databaseId)
));
} catch (error) { return safeError(reply, error); }
});
app.patch("/catalog/databases/:databaseId/relationships/:relationshipId", async (request, reply) => {
if (!manage(request, reply)) return reply;
try {
const params = request.params as { databaseId?: unknown; relationshipId?: unknown };
const databaseId = idSchema.parse(params.databaseId);
const relationshipId = idSchema.parse(params.relationshipId);
const input = statusSchema.parse(request.body);
return await deps.operations.run(databaseId, async () => (
await deps.service.setStatus(databaseId, relationshipId, input.status)
));
} catch (error) { return safeError(reply, error); }
});
app.delete("/catalog/databases/:databaseId/relationships/:relationshipId", async (request, reply) => {
if (!manage(request, reply)) return reply;
try {
const params = request.params as { databaseId?: unknown; relationshipId?: unknown };
const databaseId = idSchema.parse(params.databaseId);
const relationshipId = idSchema.parse(params.relationshipId);
await deps.operations.run(databaseId, async () => {
await deps.service.deletePermanently(databaseId, relationshipId);
});
return reply.code(204).send();
} catch (error) { return safeError(reply, error); }
});
}
-11
View File
@@ -131,17 +131,6 @@ export function catalogSchemaRoutes(
} catch (error) { return safeError(reply, error); }
});
app.get("/catalog/databases/:databaseId/relationships", async (request, reply) => {
if (!manage(request, reply)) return reply;
try {
const databaseId = idSchema.parse((request.params as { databaseId?: unknown }).databaseId);
if (!(await deps.repository.get(databaseId))) {
return reply.code(404).send({ code: "database_not_found", message: "Database configuration was not found." });
}
return await deps.repository.listRelationships(databaseId);
} catch (error) { return safeError(reply, error); }
});
app.post("/catalog/databases/metadata-cleanup", async (request, reply) => {
if (!manage(request, reply)) return reply;
try {
+35 -8
View File
@@ -11,6 +11,7 @@ import type { WorkspaceRegistry } from "../workspaces/registry.js";
import { validateOperationalWorkspace, type WorkspaceDescriptor } from "../workspaces/schema.js";
import type { MaintenanceBarrier } from "../runtime/maintenance-gate.js";
import { hasPermission, isPrincipalContext, requirePermission } from "../auth/authorization.js";
import type { EffectiveRelationshipSnapshotProvider } from "../catalog/effective-relationship-snapshot.js";
const BOOTSTRAP_FAILURE_MESSAGE =
"Session startup failed. Check configuration and connectivity, then Resume the session.";
@@ -40,6 +41,8 @@ export function sessionRoutes(
/** Fail-closed installation/runtime transport capability check. */
workspaceRuntimeSupport: (workspace: WorkspaceDescriptor) => boolean;
maintenanceBarrier: MaintenanceBarrier;
/** Optional only for narrow route-test stubs and installations without a Catalog database. */
effectiveRelationships?: EffectiveRelationshipSnapshotProvider;
},
) {
const lifecycleTails = new Map<string, Promise<void>>();
@@ -84,11 +87,21 @@ export function sessionRoutes(
isAdmin: hasPermission(principal, permission),
});
const optionsWithRuntimeConfig = (runner: any, workspaceConfigPath: string | undefined, options: any) => (
workspaceConfigPath && typeof runner.acquireWorkspaceRuntime === "function"
? { ...options, runtimeConfig: runner.acquireWorkspaceRuntime(workspaceConfigPath) }
: options
);
const optionsWithRuntimeConfig = async (
runner: any,
workspaceConfigPath: string | undefined,
workspaceId: string | undefined,
options: any,
) => {
if (!workspaceConfigPath || typeof runner.acquireWorkspaceRuntime !== "function") return options;
const effectiveRelationships = workspaceId && d.effectiveRelationships
? await d.effectiveRelationships.render(workspaceId)
: undefined;
return {
...options,
runtimeConfig: runner.acquireWorkspaceRuntime(workspaceConfigPath, effectiveRelationships),
};
};
const maintenanceReply = (reply: any) => reply.code(503).send({
code: "maintenance",
@@ -448,7 +461,12 @@ export function sessionRoutes(
let runtimeOptions = options;
let rt: ReturnType<PiProcessManager["createFor"]> | undefined;
try {
runtimeOptions = optionsWithRuntimeConfig(runner, workspaceConfigPath, options);
runtimeOptions = await optionsWithRuntimeConfig(
runner,
workspaceConfigPath,
workspaceId,
options,
);
rt = d.mgr.createFor(id, runtimeOptions);
bindRuntime(id, rt, runner, workspaceConfigPath);
} catch (error) {
@@ -465,7 +483,11 @@ export function sessionRoutes(
info(id, "Session created");
bootstrap(
id, rt, runner, workspaceConfigPath, d.mgr.configure(rt, runtimeOptions),
runner.searchPack(b.question, id, workspaceConfigPath),
runner.searchPack(
b.question,
id,
(runtimeOptions as any).runtimeConfig?.path ?? workspaceConfigPath,
),
() => d.mgr.start(id, rt, runtimeOptions),
);
return { id };
@@ -639,7 +661,12 @@ export function sessionRoutes(
if (boundRuntimes.get(id) === current) boundRuntimes.delete(id);
d.mgr.teardownIfCurrent(id, current);
}
runtimeOptions = optionsWithRuntimeConfig(runner, workspaceConfigPath, options);
runtimeOptions = await optionsWithRuntimeConfig(
runner,
workspaceConfigPath,
saved.workspace_id,
options,
);
rt = d.mgr.createFor(id, runtimeOptions);
bindRuntime(id, rt, runner, workspaceConfigPath);
} catch {
+27 -12
View File
@@ -213,23 +213,37 @@ export class ThtRunner {
}
/** Render one immutable canonical registry revision into a backend-owned harness config. */
acquireWorkspaceRuntime(workspaceConfigPath: string): RuntimeConfigLease {
const rendered = renderWorkspaceRuntimeFromSnapshotPath({
snapshotPath: workspaceConfigPath,
harnessDir: this.cfg.harnessDir,
configPath: this.cfg.configPath,
dataRoot: this.cfg.dataRoot ?? (() => {
throw new Error("registry workspace runtime requires an absolute data root");
})(),
secretRoots: this.cfg.secretRoots ?? [],
semanticRuntime: this.cfg.semanticRuntime ?? DEFAULT_SEMANTIC_RUNTIME,
workspaceSecretStore: this.cfg.workspaceSecretStore,
});
acquireWorkspaceRuntime(
workspaceConfigPath: string,
effectiveRelationships?: string,
): RuntimeConfigLease {
const effectiveRelationshipsPath = effectiveRelationships === undefined
? undefined
: this.createRuntimeSnapshot(effectiveRelationships);
let rendered: ReturnType<typeof renderWorkspaceRuntimeFromSnapshotPath>;
try {
rendered = renderWorkspaceRuntimeFromSnapshotPath({
snapshotPath: workspaceConfigPath,
harnessDir: this.cfg.harnessDir,
configPath: this.cfg.configPath,
dataRoot: this.cfg.dataRoot ?? (() => {
throw new Error("registry workspace runtime requires an absolute data root");
})(),
secretRoots: this.cfg.secretRoots ?? [],
semanticRuntime: this.cfg.semanticRuntime ?? DEFAULT_SEMANTIC_RUNTIME,
workspaceSecretStore: this.cfg.workspaceSecretStore,
effectiveRelationshipsPath,
});
} catch (error) {
if (effectiveRelationshipsPath) this.cleanupRuntimeSnapshot(effectiveRelationshipsPath);
throw error;
}
let path: string;
try {
path = this.createRuntimeSnapshot(rendered.renderedConfig);
} catch (error) {
rendered.releaseSecrets();
if (effectiveRelationshipsPath) this.cleanupRuntimeSnapshot(effectiveRelationshipsPath);
throw error;
}
let released = false;
@@ -241,6 +255,7 @@ export class ThtRunner {
if (released) return;
released = true;
this.cleanupRuntimeSnapshot(path);
if (effectiveRelationshipsPath) this.cleanupRuntimeSnapshot(effectiveRelationshipsPath);
rendered.releaseSecrets();
},
};
+24 -3
View File
@@ -243,7 +243,12 @@ function readSnapshotWorkspace(snapshotPath: string): {
}
}
function runtimePaths(dataRoot: string, workspaceId: string, workspaceRevision?: string): RuntimePaths {
function runtimePaths(
dataRoot: string,
workspaceId: string,
workspaceRevision?: string,
effectiveRelationshipsPath?: string,
): RuntimePaths {
if (!isAbsolute(dataRoot)) throw new Error("registry workspace runtime requires an absolute data root");
const root = join(dataRoot, "sessions", workspaceId);
return {
@@ -254,6 +259,9 @@ function runtimePaths(dataRoot: string, workspaceId: string, workspaceRevision?:
...(workspaceRevision === undefined
? {}
: { annotations_root: join(dataRoot, "sessions", workspaceId, "revisions", workspaceRevision, "artifacts") }),
...(effectiveRelationshipsPath === undefined
? {}
: { effective_relationships: effectiveRelationshipsPath }),
};
}
@@ -301,6 +309,7 @@ function renderWorkspaceRuntimeFromWorkspace(options: {
dataRoot: string;
secretRoots: readonly string[];
semanticRuntime: SemanticRuntimeConfig;
effectiveRelationshipsPath?: string;
workspaceSecretStore?: WorkspaceSecretStore;
}): RenderedWorkspaceRuntime {
const secretLease = options.workspaceSecretStore === undefined
@@ -325,7 +334,12 @@ function renderWorkspaceRuntimeFromWorkspace(options: {
workspaceId: options.workspaceId,
workspaceRevision: options.workspaceRevision,
revisionContentRoot: options.revisionContentRoot,
runtimePaths: runtimePaths(options.dataRoot, options.workspaceId, options.workspaceRevision),
runtimePaths: runtimePaths(
options.dataRoot,
options.workspaceId,
options.workspaceRevision,
options.effectiveRelationshipsPath,
),
installationOverlay: overlay,
bindings,
bindingDigest: stableBindingDigest(bindings),
@@ -334,7 +348,12 @@ function renderWorkspaceRuntimeFromWorkspace(options: {
renderedConfig: renderRuntimeConfig(
options.workspace,
bindings,
runtimePaths(options.dataRoot, options.workspaceId, options.workspaceRevision),
runtimePaths(
options.dataRoot,
options.workspaceId,
options.workspaceRevision,
options.effectiveRelationshipsPath,
),
context,
overlay,
options.semanticRuntime,
@@ -353,6 +372,7 @@ export function renderWorkspaceRuntimeFromSnapshotPath(options: {
dataRoot: string;
secretRoots: readonly string[];
semanticRuntime: SemanticRuntimeConfig;
effectiveRelationshipsPath?: string;
workspaceSecretStore?: WorkspaceSecretStore;
}): RenderedWorkspaceRuntime {
const snapshot = readSnapshotWorkspace(options.snapshotPath);
@@ -366,6 +386,7 @@ export function renderWorkspaceRuntimeFromSnapshotPath(options: {
dataRoot: options.dataRoot,
secretRoots: options.secretRoots,
semanticRuntime: options.semanticRuntime,
effectiveRelationshipsPath: options.effectiveRelationshipsPath,
workspaceSecretStore: options.workspaceSecretStore,
});
}
@@ -12,6 +12,8 @@ export interface RuntimePaths {
memory: string;
/** Revision-qualified root for curated FK annotations (P5); optional for legacy callers. */
annotations_root?: string;
/** Immutable Catalog projection used as the exclusive runtime relationship source. */
effective_relationships?: string;
}
export interface RuntimeIdentity {