feat: consolidate database management work

Add catalog-owned logical relationships and runtime snapshots, extend the database-management UI and validation coverage, and document the updated operational workflow.

Keep active sensitive-generation status in a tooltip and indicator, and update the layout E2E to follow the history action in its new database-scoped location.
This commit is contained in:
Codex
2026-09-01 14:46:55 +02:00
parent f586152636
commit 076c9742c5
73 changed files with 6966 additions and 610 deletions
+20
View File
@@ -63,6 +63,9 @@ import {
type DescriptionSourceSampler,
} from "./catalog/description-source-sampler.js";
import { catalogDescriptionGenerationRoutes } from "./routes/catalog-description-generation.js";
import { CatalogLogicalRelationshipService } from "./catalog/logical-relationship-service.js";
import { catalogLogicalRelationshipRoutes } from "./routes/catalog-logical-relationships.js";
import { EffectiveRelationshipSnapshotProvider } from "./catalog/effective-relationship-snapshot.js";
export interface BuildAppDeps {
thtRunner?: ThtRunner;
@@ -79,6 +82,8 @@ export interface BuildAppDeps {
catalogService?: CatalogService;
catalogPostgresAccess?: CatalogPostgresAccess;
catalogTableService?: CatalogTableService;
catalogLogicalRelationshipService?: CatalogLogicalRelationshipService;
effectiveRelationshipSnapshotProvider?: EffectiveRelationshipSnapshotProvider;
catalogSchemaIntrospector?: CatalogSchemaIntrospector;
catalogSyncWorker?: CatalogSyncWorker;
catalogOperationCoordinator?: CatalogOperationCoordinator;
@@ -200,6 +205,16 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
catalogOperationCoordinator,
);
const catalogTableService = deps?.catalogTableService ?? new CatalogTableService(catalogRepository);
const catalogLogicalRelationshipService = deps?.catalogLogicalRelationshipService
?? new CatalogLogicalRelationshipService(catalogRepository);
const effectiveRelationships = deps?.effectiveRelationshipSnapshotProvider
?? (config.catalogDatabase === undefined
? undefined
: new EffectiveRelationshipSnapshotProvider(
catalogRepository,
catalogLogicalRelationshipService,
catalogOperationCoordinator,
));
const catalogSchemaIntrospector = deps?.catalogSchemaIntrospector ?? new ConcreteCatalogSchemaIntrospector(
catalogPostgresAccess,
workspaceSecretStore,
@@ -394,6 +409,7 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
legacyWorkspaceMode: config.legacyWorkspaceMode,
workspaceRuntimeSupport,
maintenanceBarrier,
effectiveRelationships,
});
app.post("/internal/maintenance/activate", async (req, reply) => {
try {
@@ -442,6 +458,10 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc
worker: catalogSyncWorker,
operations: catalogOperationCoordinator,
});
catalogLogicalRelationshipRoutes(app, {
service: catalogLogicalRelationshipService,
operations: catalogOperationCoordinator,
});
catalogDescriptionConsolidationRoutes(app, {
repository: catalogRepository,
operations: catalogOperationCoordinator,
@@ -0,0 +1,93 @@
import type { CatalogRelationship, CatalogRepository } from "./types.js";
export interface EffectiveRelationshipSnapshotReader {
list(databaseId: string): Promise<CatalogRelationship[]>;
}
export interface EffectiveRelationshipSnapshotCoordinator {
run<T>(databaseId: string, operation: () => Promise<T>): Promise<T>;
}
export class EffectiveRelationshipSnapshotStaleError extends Error {}
interface EffectiveRelationship {
sourceTable: string;
sourceColumns: string[];
targetTable: string;
targetColumns: string[];
origin: CatalogRelationship["origin"];
}
interface EffectiveRelationshipSnapshot {
schemaVersion: 1;
workspaceId: string;
relationships: EffectiveRelationship[];
}
const originRank: Record<CatalogRelationship["origin"], number> = {
physical: 0,
manual: 1,
generated: 2,
};
function endpointKey(relationship: EffectiveRelationship): string {
return [
relationship.sourceTable,
relationship.sourceColumns.join("\u0000"),
relationship.targetTable,
relationship.targetColumns.join("\u0000"),
].join("\u0001");
}
function compareRelationships(left: EffectiveRelationship, right: EffectiveRelationship): number {
return endpointKey(left).localeCompare(endpointKey(right))
|| originRank[left.origin] - originRank[right.origin];
}
/**
* Adapter from the mutable Catalog model to the immutable relationship contract consumed by the
* harness. The returned JSON is a deterministic projection, never an authored second store.
*/
export class EffectiveRelationshipSnapshotProvider {
constructor(
private readonly repository: Pick<CatalogRepository, "get" | "getByWorkspace">,
private readonly relationships: EffectiveRelationshipSnapshotReader,
private readonly operations: EffectiveRelationshipSnapshotCoordinator,
) {}
async render(workspaceId: string): Promise<string | undefined> {
const database = await this.repository.getByWorkspace(workspaceId);
if (!database) return undefined;
return await this.operations.run(database.id, async () => {
const current = await this.repository.get(database.id);
if (!current || current.schemaSyncedVersion !== current.version) {
throw new EffectiveRelationshipSnapshotStaleError(
"effective relationship snapshot requires a current full schema synchronization",
);
}
const projected = (await this.relationships.list(database.id))
.filter((relationship) => relationship.status === "active")
.map((relationship): EffectiveRelationship => ({
sourceTable: relationship.sourceTableName,
sourceColumns: relationship.columns.map((column) => column.sourceColumnName),
targetTable: relationship.targetTableName,
targetColumns: relationship.columns.map((column) => column.targetColumnName),
origin: relationship.origin,
}))
.sort(compareRelationships);
const seen = new Set<string>();
const snapshot: EffectiveRelationshipSnapshot = {
schemaVersion: 1,
workspaceId,
relationships: projected.filter((relationship) => {
const key = endpointKey(relationship);
if (seen.has(key)) return false;
seen.add(key);
return true;
}),
};
return `${JSON.stringify(snapshot, null, 2)}\n`;
});
}
}
@@ -0,0 +1,260 @@
import type {
CatalogLogicalRelationship,
CatalogLogicalRelationshipCandidate,
CatalogLogicalRelationshipContext,
CatalogLogicalRelationshipEndpoint,
CatalogRelationship,
CatalogRepository,
} from "./types.js";
export class LogicalRelationshipDatabaseNotFoundError extends Error {}
export class LogicalRelationshipDuplicateError extends Error {}
export class LogicalRelationshipNotFoundError extends Error {}
export class LogicalRelationshipReadOnlyError extends Error {}
export class LogicalRelationshipSchemaStaleError extends Error {}
export class LogicalRelationshipTargetNotUniqueError extends Error {}
export class LogicalRelationshipTypeIncompatibleError extends Error {}
export class LogicalRelationshipColumnNotFoundError extends Error {
constructor(readonly field: "sourceColumnId" | "targetColumnId") {
super(`Catalog column '${field}' was not found`);
}
}
export interface RebuildGeneratedRelationshipsResult {
added: number;
alreadyPresent: number;
excluded: number;
ambiguous: number;
}
function identifierTokens(value: string): string[] {
return value
.replace(/([a-z0-9])([A-Z])/g, "$1_$2")
.toLowerCase()
.split(/[^a-z0-9]+/)
.filter(Boolean);
}
function singularWord(value: string): string {
if (value.length > 4 && value.endsWith("ies")) return `${value.slice(0, -3)}y`;
if (value.length > 4 && /(ches|shes|xes|zes|ses)$/.test(value)) return value.slice(0, -2);
if (value.length > 3 && value.endsWith("s") && !/(ss|us)$/.test(value)) return value.slice(0, -1);
return value;
}
function tableAliases(tableName: string): string[] {
const tokens = identifierTokens(tableName);
if (tokens.length === 0) return [];
const normalized = tokens.join("_");
const singular = [...tokens];
singular[singular.length - 1] = singularWord(singular[singular.length - 1]);
return [...new Set([normalized, singular.join("_")])];
}
const GENERIC_PRIMARY_KEY_NAMES = new Set(["id", "key", "code", "pk"]);
function nameMatches(
source: CatalogLogicalRelationshipEndpoint,
target: CatalogLogicalRelationshipEndpoint,
): boolean {
const sourceName = identifierTokens(source.columnName).join("_");
const targetName = identifierTokens(target.columnName).join("_");
if (!sourceName || !targetName) return false;
const expected = new Set<string>();
if (!GENERIC_PRIMARY_KEY_NAMES.has(targetName)) expected.add(targetName);
for (const alias of tableAliases(target.tableName)) {
expected.add(`${alias}_${targetName}`);
expected.add(`${alias.replaceAll("_", "")}${targetName.replaceAll("_", "")}`);
if (targetName === "id" || targetName === "pk") expected.add(alias);
}
return expected.has(sourceName);
}
function canonicalDataType(value: string): string {
const normalized = value.trim().toLowerCase().replace(/\s+/g, " ");
const arraySuffix = normalized.endsWith("[]") ? "[]" : "";
const base = arraySuffix ? normalized.slice(0, -2) : normalized;
const withoutModifier = base.replace(/\([^)]*\)/g, "").trim();
const aliases: Record<string, string> = {
int2: "smallint",
smallserial: "smallint",
int4: "integer",
int: "integer",
serial: "integer",
int8: "bigint",
bigserial: "bigint",
decimal: "numeric",
varchar: "text",
"character varying": "text",
bool: "boolean",
"timestamp without time zone": "timestamp",
"timestamp with time zone": "timestamptz",
"time without time zone": "time",
"time with time zone": "timetz",
};
return `${aliases[withoutModifier] ?? withoutModifier}${arraySuffix}`;
}
function typesCompatible(left: string, right: string): boolean {
return canonicalDataType(left) === canonicalDataType(right);
}
function pairKey(sourceColumnId: string, targetColumnId: string): string {
return `${sourceColumnId}\u0000${targetColumnId}`;
}
function relationshipPair(relationship: CatalogLogicalRelationship): CatalogLogicalRelationshipCandidate {
return {
sourceColumnId: relationship.columns[0].sourceColumnId,
targetColumnId: relationship.columns[0].targetColumnId,
};
}
function relationshipSortKey(relationship: CatalogRelationship): string {
const sourceColumns = relationship.columns.map((column) => column.sourceColumnName).join(",");
const targetColumns = relationship.columns.map((column) => column.targetColumnName).join(",");
return [
relationship.sourceTableName,
sourceColumns,
relationship.targetTableName,
targetColumns,
relationship.origin,
].join("\u0000");
}
export class CatalogLogicalRelationshipService {
constructor(private readonly repository: CatalogRepository) {}
async list(databaseId: string): Promise<CatalogRelationship[]> {
if (!(await this.repository.get(databaseId))) throw new LogicalRelationshipDatabaseNotFoundError();
const relationships: CatalogRelationship[] = [
...await this.repository.listRelationships(databaseId),
...await this.repository.listLogicalRelationships(databaseId),
];
return relationships.sort((left, right) => relationshipSortKey(left).localeCompare(relationshipSortKey(right)));
}
async addManual(
databaseId: string,
sourceColumnId: string,
targetColumnId: string,
): Promise<CatalogLogicalRelationship> {
const context = await this.requiredContext(databaseId);
const source = context.endpoints.find((endpoint) => endpoint.columnId === sourceColumnId);
if (!source) throw new LogicalRelationshipColumnNotFoundError("sourceColumnId");
const target = context.endpoints.find((endpoint) => endpoint.columnId === targetColumnId);
if (!target) throw new LogicalRelationshipColumnNotFoundError("targetColumnId");
if (sourceColumnId === targetColumnId
|| target.primaryKeyPosition === null
|| target.tablePrimaryKeyColumnCount !== 1) {
throw new LogicalRelationshipTargetNotUniqueError();
}
if (!typesCompatible(source.dataType, target.dataType)) {
throw new LogicalRelationshipTypeIncompatibleError();
}
const key = pairKey(sourceColumnId, targetColumnId);
if (context.physicalPairs.some((pair) => pairKey(pair.sourceColumnId, pair.targetColumnId) === key)
|| context.logicalRelationships.some((relationship) => {
const pair = relationshipPair(relationship);
return pairKey(pair.sourceColumnId, pair.targetColumnId) === key;
})) throw new LogicalRelationshipDuplicateError();
const created = await this.repository.insertLogicalRelationship(
databaseId,
sourceColumnId,
targetColumnId,
false,
);
if (!created) throw new LogicalRelationshipDuplicateError();
return created;
}
async rebuildGenerated(databaseId: string): Promise<RebuildGeneratedRelationshipsResult> {
const context = await this.requiredContext(databaseId);
const targets = context.endpoints.filter((endpoint) => (
endpoint.primaryKeyPosition !== null && endpoint.tablePrimaryKeyColumnCount === 1
));
const physical = new Set(context.physicalPairs.map((pair) => pairKey(pair.sourceColumnId, pair.targetColumnId)));
const active = new Set<string>();
const excluded = new Set<string>();
for (const relationship of context.logicalRelationships) {
const pair = relationshipPair(relationship);
(relationship.status === "excluded" ? excluded : active)
.add(pairKey(pair.sourceColumnId, pair.targetColumnId));
}
const pending: CatalogLogicalRelationshipCandidate[] = [];
let alreadyPresent = 0;
let excludedCount = 0;
let ambiguous = 0;
const dimTimeTargets = targets.filter((target) => (
identifierTokens(target.tableName).join("_") === "dim_time"
));
const sources = context.endpoints.filter((endpoint) => (
endpoint.primaryKeyPosition === null || endpoint.tablePrimaryKeyColumnCount > 1
));
for (const source of sources) {
const sourceName = identifierTokens(source.columnName).join("_");
const isTimeKey = sourceName.endsWith("time_key")
&& identifierTokens(source.tableName).join("_") !== "dim_time";
const candidates = isTimeKey ? dimTimeTargets : targets;
const matches = candidates.filter((target) => (
target.columnId !== source.columnId
&& typesCompatible(source.dataType, target.dataType)
&& (isTimeKey || nameMatches(source, target))
));
if (matches.length > 1) {
ambiguous += 1;
continue;
}
if (matches.length === 0) continue;
const candidate = { sourceColumnId: source.columnId, targetColumnId: matches[0].columnId };
const key = pairKey(candidate.sourceColumnId, candidate.targetColumnId);
if (physical.has(key) || active.has(key)) {
alreadyPresent += 1;
} else if (excluded.has(key)) {
excludedCount += 1;
} else {
pending.push(candidate);
}
}
const added = await this.repository.insertGeneratedLogicalRelationships(databaseId, pending);
alreadyPresent += pending.length - added;
return { added, alreadyPresent, excluded: excludedCount, ambiguous };
}
async setStatus(
databaseId: string,
relationshipId: string,
status: CatalogLogicalRelationship["status"],
): Promise<CatalogLogicalRelationship> {
if (!(await this.repository.get(databaseId))) throw new LogicalRelationshipDatabaseNotFoundError();
const updated = await this.repository.setLogicalRelationshipStatus(databaseId, relationshipId, status);
if (updated) return updated;
await this.assertNotPhysical(databaseId, relationshipId);
throw new LogicalRelationshipNotFoundError();
}
async deletePermanently(databaseId: string, relationshipId: string): Promise<void> {
if (!(await this.repository.get(databaseId))) throw new LogicalRelationshipDatabaseNotFoundError();
if (await this.repository.deleteLogicalRelationship(databaseId, relationshipId)) return;
await this.assertNotPhysical(databaseId, relationshipId);
throw new LogicalRelationshipNotFoundError();
}
private async requiredContext(databaseId: string): Promise<CatalogLogicalRelationshipContext> {
const database = await this.repository.get(databaseId);
if (!database) throw new LogicalRelationshipDatabaseNotFoundError();
if (database.schemaSyncedVersion !== database.version) {
throw new LogicalRelationshipSchemaStaleError();
}
const context = await this.repository.getLogicalRelationshipContext(databaseId);
if (!context) throw new LogicalRelationshipDatabaseNotFoundError();
return context;
}
private async assertNotPhysical(databaseId: string, relationshipId: string): Promise<void> {
if ((await this.repository.listRelationships(databaseId)).some((relationship) => relationship.id === relationshipId)) {
throw new LogicalRelationshipReadOnlyError();
}
}
}
+150 -3
View File
@@ -14,7 +14,10 @@ import {
type CatalogDatabaseMetadataDeleteTarget,
type CatalogMetadataDeleteCounts,
type CatalogMetrics,
type CatalogRelationship,
type CatalogLogicalRelationship,
type CatalogLogicalRelationshipCandidate,
type CatalogLogicalRelationshipContext,
type CatalogPhysicalRelationship,
type CatalogSchemaDiff,
type CatalogSyncCounts,
type CatalogSyncEvent,
@@ -49,7 +52,8 @@ export class MemoryCatalogRepository implements CatalogRepository {
private readonly records = new Map<string, WorkspaceDatabase>();
private readonly tables = new Map<string, CatalogTable>();
private readonly columns = new Map<string, CatalogColumn>();
private readonly relationships = new Map<string, CatalogRelationship>();
private readonly relationships = new Map<string, CatalogPhysicalRelationship>();
private readonly logicalRelationships = new Map<string, CatalogLogicalRelationship>();
private readonly descriptionGenerationRuns = new Map<string, DescriptionGenerationRun>();
private readonly descriptionGenerationEvents = new Map<string, DescriptionGenerationEvent[]>();
private readonly sensitiveDataSuggestionRuns = new Map<string, SensitiveDataSuggestionRun>();
@@ -171,6 +175,9 @@ export class MemoryCatalogRepository implements CatalogRepository {
for (const [relationshipId, relationship] of this.relationships) {
if (relationship.databaseId === id) this.relationships.delete(relationshipId);
}
for (const [relationshipId, relationship] of this.logicalRelationships) {
if (relationship.databaseId === id) this.logicalRelationships.delete(relationshipId);
}
for (const [runId, run] of this.descriptionGenerationRuns) {
if (run.databaseId !== id) continue;
this.descriptionGenerationRuns.delete(runId);
@@ -527,12 +534,138 @@ export class MemoryCatalogRepository implements CatalogRepository {
.map((event) => structuredClone(event));
}
async listRelationships(databaseId: string): Promise<CatalogRelationship[]> {
async listRelationships(databaseId: string): Promise<CatalogPhysicalRelationship[]> {
return [...this.relationships.values()].filter((relationship) => relationship.databaseId === databaseId)
.sort((a, b) => `${a.sourceTableName}.${a.constraintName}`.localeCompare(`${b.sourceTableName}.${b.constraintName}`))
.map((relationship) => structuredClone(relationship));
}
async listLogicalRelationships(databaseId: string): Promise<CatalogLogicalRelationship[]> {
return [...this.logicalRelationships.values()]
.filter((relationship) => relationship.databaseId === databaseId)
.sort((a, b) => {
const left = `${a.sourceTableName}.${a.columns[0].sourceColumnName}.${a.targetTableName}.${a.columns[0].targetColumnName}`;
const right = `${b.sourceTableName}.${b.columns[0].sourceColumnName}.${b.targetTableName}.${b.columns[0].targetColumnName}`;
return left.localeCompare(right);
})
.map((relationship) => structuredClone(relationship));
}
async getLogicalRelationshipContext(
databaseId: string,
): Promise<CatalogLogicalRelationshipContext | undefined> {
if (!this.records.has(databaseId)) return undefined;
const tables = [...this.tables.values()].filter((table) => table.databaseId === databaseId);
const tableById = new Map(tables.map((table) => [table.id, table]));
const columns = [...this.columns.values()].filter((column) => tableById.has(column.tableId));
const primaryKeyCounts = new Map<string, number>();
for (const column of columns) {
if (column.primaryKeyPosition !== null) {
primaryKeyCounts.set(column.tableId, (primaryKeyCounts.get(column.tableId) ?? 0) + 1);
}
}
return {
endpoints: columns.map((column) => ({
columnId: column.id,
columnName: column.name,
tableId: column.tableId,
tableName: tableById.get(column.tableId)!.name,
dataType: column.dataType,
primaryKeyPosition: column.primaryKeyPosition,
tablePrimaryKeyColumnCount: primaryKeyCounts.get(column.tableId) ?? 0,
})),
physicalPairs: [...this.relationships.values()]
.filter((relationship) => relationship.databaseId === databaseId)
.flatMap((relationship) => relationship.columns.map((column) => ({
sourceColumnId: column.sourceColumnId,
targetColumnId: column.targetColumnId,
}))),
logicalRelationships: await this.listLogicalRelationships(databaseId),
};
}
async insertLogicalRelationship(
databaseId: string,
sourceColumnId: string,
targetColumnId: string,
generated: boolean,
): Promise<CatalogLogicalRelationship | undefined> {
if ([...this.logicalRelationships.values()].some((relationship) => (
relationship.databaseId === databaseId
&& relationship.columns[0].sourceColumnId === sourceColumnId
&& relationship.columns[0].targetColumnId === targetColumnId
))) return undefined;
const sourceColumn = this.columns.get(sourceColumnId);
const targetColumn = this.columns.get(targetColumnId);
const sourceTable = sourceColumn ? this.tables.get(sourceColumn.tableId) : undefined;
const targetTable = targetColumn ? this.tables.get(targetColumn.tableId) : undefined;
if (!sourceColumn || !targetColumn || !sourceTable || !targetTable
|| sourceTable.databaseId !== databaseId || targetTable.databaseId !== databaseId
|| sourceColumnId === targetColumnId) return undefined;
const now = new Date().toISOString();
const relationship: CatalogLogicalRelationship = {
id: randomUUID(),
databaseId,
constraintName: null,
sourceTableId: sourceTable.id,
sourceTableName: sourceTable.name,
targetTableId: targetTable.id,
targetTableName: targetTable.name,
updateRule: null,
deleteRule: null,
deferrable: false,
initiallyDeferred: false,
columns: [{
position: 1,
sourceColumnId,
sourceColumnName: sourceColumn.name,
targetColumnId,
targetColumnName: targetColumn.name,
}],
lastSyncedDatabaseVersion: null,
lastSyncedAt: null,
createdAt: now,
updatedAt: now,
origin: generated ? "generated" : "manual",
status: "active",
};
this.logicalRelationships.set(relationship.id, relationship);
return structuredClone(relationship);
}
async insertGeneratedLogicalRelationships(
databaseId: string,
candidates: readonly CatalogLogicalRelationshipCandidate[],
): Promise<number> {
let added = 0;
for (const candidate of candidates) {
if (await this.insertLogicalRelationship(
databaseId,
candidate.sourceColumnId,
candidate.targetColumnId,
true,
)) added += 1;
}
return added;
}
async setLogicalRelationshipStatus(
databaseId: string,
relationshipId: string,
status: CatalogLogicalRelationship["status"],
): Promise<CatalogLogicalRelationship | undefined> {
const current = this.logicalRelationships.get(relationshipId);
if (!current || current.databaseId !== databaseId) return undefined;
const updated = { ...current, status, updatedAt: new Date().toISOString() };
this.logicalRelationships.set(relationshipId, updated);
return structuredClone(updated);
}
async deleteLogicalRelationship(databaseId: string, relationshipId: string): Promise<boolean> {
const current = this.logicalRelationships.get(relationshipId);
return Boolean(current?.databaseId === databaseId && this.logicalRelationships.delete(relationshipId));
}
async deleteDatabaseMetadata(
databaseIds: readonly string[],
target: CatalogDatabaseMetadataDeleteTarget,
@@ -574,6 +707,12 @@ export class MemoryCatalogRepository implements CatalogRepository {
const columns = [...this.columns.values()].filter((column) => selected.has(column.tableId));
const deletedColumnIds = new Set(columns.map((column) => column.id));
for (const column of columns) this.columns.delete(column.id);
for (const relationship of [...this.logicalRelationships.values()]) {
const pair = relationship.columns[0];
if (deletedColumnIds.has(pair.sourceColumnId) || deletedColumnIds.has(pair.targetColumnId)) {
this.logicalRelationships.delete(relationship.id);
}
}
for (const [relationshipId, relationship] of this.relationships) {
if (relationship.databaseId !== databaseId) continue;
this.relationships.set(relationshipId, {
@@ -857,6 +996,8 @@ export class MemoryCatalogRepository implements CatalogRepository {
lastSyncedAt: now,
createdAt: current?.createdAt ?? now,
updatedAt: comparable === nextComparable ? (current?.updatedAt ?? now) : now,
origin: "physical",
status: "active",
});
if (!current) created += 1;
else if (comparable !== nextComparable) updated += 1;
@@ -1073,6 +1214,12 @@ export class MemoryCatalogRepository implements CatalogRepository {
this.relationships.delete(relationship.id);
}
}
for (const relationship of [...this.logicalRelationships.values()]) {
const pair = relationship.columns[0];
if (pair.sourceColumnId === columnId || pair.targetColumnId === columnId) {
this.logicalRelationships.delete(relationship.id);
}
}
}
private markCatalogIncomplete(databaseIds: readonly string[]): void {
+2
View File
@@ -10,6 +10,7 @@ import * as catalogRuntimeSequencePrivilegesMigration from "./migrations/004_cat
import * as descriptionGenerationRunsMigration from "./migrations/005_description_generation_runs.js";
import * as sensitiveDataFlagMigration from "./migrations/006_sensitive_data_flag.js";
import * as sensitiveDataSuggestionRunsMigration from "./migrations/007_sensitive_data_suggestion_runs.js";
import * as catalogLogicalRelationshipsMigration from "./migrations/008_catalog_logical_relationships.js";
const connectionString = process.env.THT_CATALOG_MIGRATOR_DATABASE_URL;
const host = process.env.THT_CATALOG_DB_HOST;
@@ -42,6 +43,7 @@ const provider: MigrationProvider = {
"005_description_generation_runs": descriptionGenerationRunsMigration,
"006_sensitive_data_flag": sensitiveDataFlagMigration,
"007_sensitive_data_suggestion_runs": sensitiveDataSuggestionRunsMigration,
"008_catalog_logical_relationships": catalogLogicalRelationshipsMigration,
};
},
};
@@ -0,0 +1,35 @@
import { type Kysely, sql } from "kysely";
import type { CatalogDatabase } from "../repository.js";
export async function up(db: Kysely<CatalogDatabase>): Promise<void> {
await db.schema.createTable("catalog_logical_relationships")
.addColumn("id", "uuid", (column) => column.primaryKey())
.addColumn("database_id", "uuid", (column) => column.notNull()
.references("workspace_databases.id").onDelete("cascade"))
.addColumn("source_column_id", "uuid", (column) => column.notNull()
.references("catalog_columns.id").onDelete("cascade"))
.addColumn("target_column_id", "uuid", (column) => column.notNull()
.references("catalog_columns.id").onDelete("cascade"))
.addColumn("generated", "boolean", (column) => column.notNull().defaultTo(false))
.addColumn("deleted_at", "timestamptz")
.addColumn("created_at", "timestamptz", (column) => column.notNull().defaultTo(sql`now()`))
.addColumn("updated_at", "timestamptz", (column) => column.notNull().defaultTo(sql`now()`))
.addUniqueConstraint(
"catalog_logical_relationships_endpoint_key",
["database_id", "source_column_id", "target_column_id"],
)
.addCheckConstraint(
"catalog_logical_relationships_distinct_columns_check",
sql`source_column_id <> target_column_id`,
)
.execute();
await db.schema.createIndex("catalog_logical_relationships_database_deleted_idx")
.on("catalog_logical_relationships")
.columns(["database_id", "deleted_at"])
.execute();
}
export async function down(db: Kysely<CatalogDatabase>): Promise<void> {
await db.schema.dropTable("catalog_logical_relationships").execute();
}
+176 -4
View File
@@ -23,7 +23,10 @@ import {
type CatalogDatabaseMetadataDeleteTarget,
type CatalogMetadataDeleteCounts,
type CatalogMetrics,
type CatalogRelationship,
type CatalogLogicalRelationship,
type CatalogLogicalRelationshipCandidate,
type CatalogLogicalRelationshipContext,
type CatalogPhysicalRelationship,
type CatalogSchemaDiff,
type CatalogSyncCounts,
type CatalogSyncEvent,
@@ -145,6 +148,17 @@ interface CatalogRelationshipColumnTable {
targetColumnId: string;
}
interface CatalogLogicalRelationshipTable {
id: string;
databaseId: string;
sourceColumnId: string;
targetColumnId: string;
generated: Generated<boolean>;
deletedAt: Timestamp | null;
createdAt: Timestamp;
updatedAt: Timestamp;
}
interface DescriptionGenerationRunTable {
id: string;
databaseId: string;
@@ -241,6 +255,7 @@ export interface CatalogDatabase {
catalogColumns: CatalogColumnTable;
catalogRelationships: CatalogRelationshipTable;
catalogRelationshipColumns: CatalogRelationshipColumnTable;
catalogLogicalRelationships: CatalogLogicalRelationshipTable;
descriptionGenerationRuns: DescriptionGenerationRunTable;
descriptionGenerationEvents: DescriptionGenerationEventTable;
sensitiveDataSuggestionRuns: SensitiveDataSuggestionRunTable;
@@ -1022,7 +1037,7 @@ export class KyselyCatalogRepository implements CatalogRepository {
return rows.map(serializeSensitiveDataSuggestionEvent);
}
async listRelationships(databaseId: string): Promise<CatalogRelationship[]> {
async listRelationships(databaseId: string): Promise<CatalogPhysicalRelationship[]> {
const rows = await this.db.selectFrom("catalogRelationships as relationship")
.innerJoin("catalogTables as sourceTable", "sourceTable.id", "relationship.sourceTableId")
.innerJoin("catalogTables as targetTable", "targetTable.id", "relationship.targetTableId")
@@ -1049,7 +1064,7 @@ export class KyselyCatalogRepository implements CatalogRepository {
.where("pair.relationshipId", "in", rows.map((row) => row.id))
.orderBy("pair.relationshipId").orderBy("pair.position")
.execute();
const byRelationship = new Map<string, CatalogRelationship["columns"]>();
const byRelationship = new Map<string, CatalogPhysicalRelationship["columns"]>();
for (const pair of pairs) {
const items = byRelationship.get(pair.relationshipId) ?? [];
items.push(pair);
@@ -1061,9 +1076,160 @@ export class KyselyCatalogRepository implements CatalogRepository {
lastSyncedAt: row.lastSyncedAt === null ? null : new Date(row.lastSyncedAt).toISOString(),
createdAt: new Date(row.createdAt).toISOString(),
updatedAt: new Date(row.updatedAt).toISOString(),
origin: "physical" as const,
status: "active" as const,
}));
}
async listLogicalRelationships(databaseId: string): Promise<CatalogLogicalRelationship[]> {
const rows = await this.db.selectFrom("catalogLogicalRelationships as relationship")
.innerJoin("catalogColumns as sourceColumn", "sourceColumn.id", "relationship.sourceColumnId")
.innerJoin("catalogTables as sourceTable", "sourceTable.id", "sourceColumn.tableId")
.innerJoin("catalogColumns as targetColumn", "targetColumn.id", "relationship.targetColumnId")
.innerJoin("catalogTables as targetTable", "targetTable.id", "targetColumn.tableId")
.select([
"relationship.id", "relationship.databaseId", "relationship.generated",
"relationship.deletedAt", "relationship.createdAt", "relationship.updatedAt",
"sourceTable.id as sourceTableId", "sourceTable.name as sourceTableName",
"sourceColumn.id as sourceColumnId", "sourceColumn.name as sourceColumnName",
"targetTable.id as targetTableId", "targetTable.name as targetTableName",
"targetColumn.id as targetColumnId", "targetColumn.name as targetColumnName",
])
.where("relationship.databaseId", "=", databaseId)
.orderBy("sourceTable.name")
.orderBy("sourceColumn.name")
.orderBy("targetTable.name")
.orderBy("targetColumn.name")
.execute();
return rows.map((row) => ({
id: row.id,
databaseId: row.databaseId,
constraintName: null,
sourceTableId: row.sourceTableId,
sourceTableName: row.sourceTableName,
targetTableId: row.targetTableId,
targetTableName: row.targetTableName,
updateRule: null,
deleteRule: null,
deferrable: false,
initiallyDeferred: false,
columns: [{
position: 1,
sourceColumnId: row.sourceColumnId,
sourceColumnName: row.sourceColumnName,
targetColumnId: row.targetColumnId,
targetColumnName: row.targetColumnName,
}],
lastSyncedDatabaseVersion: null,
lastSyncedAt: null,
createdAt: new Date(row.createdAt).toISOString(),
updatedAt: new Date(row.updatedAt).toISOString(),
origin: row.generated ? "generated" : "manual",
status: row.deletedAt === null ? "active" : "excluded",
}));
}
async getLogicalRelationshipContext(
databaseId: string,
): Promise<CatalogLogicalRelationshipContext | undefined> {
if (!(await selectOne(this.db, databaseId))) return undefined;
const columns = await this.db.selectFrom("catalogColumns as column")
.innerJoin("catalogTables as table", "table.id", "column.tableId")
.select([
"column.id as columnId", "column.name as columnName", "column.dataType",
"column.primaryKeyPosition", "table.id as tableId", "table.name as tableName",
])
.where("table.databaseId", "=", databaseId)
.orderBy("table.name")
.orderBy("column.ordinalPosition")
.execute();
const primaryKeyCounts = new Map<string, number>();
for (const column of columns) {
if (column.primaryKeyPosition !== null) {
primaryKeyCounts.set(column.tableId, (primaryKeyCounts.get(column.tableId) ?? 0) + 1);
}
}
const physicalPairs = await this.db.selectFrom("catalogRelationshipColumns as pair")
.innerJoin("catalogRelationships as relationship", "relationship.id", "pair.relationshipId")
.select(["pair.sourceColumnId", "pair.targetColumnId"])
.where("relationship.databaseId", "=", databaseId)
.execute();
return {
endpoints: columns.map((column) => ({
...column,
tablePrimaryKeyColumnCount: primaryKeyCounts.get(column.tableId) ?? 0,
})),
physicalPairs,
logicalRelationships: await this.listLogicalRelationships(databaseId),
};
}
async insertLogicalRelationship(
databaseId: string,
sourceColumnId: string,
targetColumnId: string,
generated: boolean,
): Promise<CatalogLogicalRelationship | undefined> {
const id = randomUUID();
const inserted = await this.db.insertInto("catalogLogicalRelationships").values({
id, databaseId, sourceColumnId, targetColumnId, generated,
}).onConflict((conflict) => conflict
.columns(["databaseId", "sourceColumnId", "targetColumnId"])
.doNothing())
.returning("id")
.executeTakeFirst();
if (!inserted) return undefined;
return (await this.listLogicalRelationships(databaseId)).find((item) => item.id === id);
}
async insertGeneratedLogicalRelationships(
databaseId: string,
candidates: readonly CatalogLogicalRelationshipCandidate[],
): Promise<number> {
if (candidates.length === 0) return 0;
return await this.db.transaction().execute(async (trx) => {
let added = 0;
for (const candidate of candidates) {
const inserted = await trx.insertInto("catalogLogicalRelationships").values({
id: randomUUID(),
databaseId,
sourceColumnId: candidate.sourceColumnId,
targetColumnId: candidate.targetColumnId,
generated: true,
}).onConflict((conflict) => conflict
.columns(["databaseId", "sourceColumnId", "targetColumnId"])
.doNothing())
.returning("id")
.executeTakeFirst();
if (inserted) added += 1;
}
return added;
});
}
async setLogicalRelationshipStatus(
databaseId: string,
relationshipId: string,
status: CatalogLogicalRelationship["status"],
): Promise<CatalogLogicalRelationship | undefined> {
const updated = await this.db.updateTable("catalogLogicalRelationships")
.set({ deletedAt: status === "excluded" ? sql`now()` : null, updatedAt: sql`now()` })
.where("databaseId", "=", databaseId)
.where("id", "=", relationshipId)
.returning("id")
.executeTakeFirst();
if (!updated) return undefined;
return (await this.listLogicalRelationships(databaseId)).find((item) => item.id === relationshipId);
}
async deleteLogicalRelationship(databaseId: string, relationshipId: string): Promise<boolean> {
const result = await this.db.deleteFrom("catalogLogicalRelationships")
.where("databaseId", "=", databaseId)
.where("id", "=", relationshipId)
.executeTakeFirst();
return result.numDeletedRows > 0n;
}
async deleteDatabaseMetadata(
databaseIds: readonly string[],
target: CatalogDatabaseMetadataDeleteTarget,
@@ -1621,7 +1787,13 @@ export class UnavailableCatalogRepository implements CatalogRepository {
async updateSensitiveDataSuggestionRun(): Promise<SensitiveDataSuggestionRun | undefined> { return this.fail(); }
async appendSensitiveDataSuggestionEvent(): Promise<SensitiveDataSuggestionEvent> { return this.fail(); }
async listSensitiveDataSuggestionEvents(): Promise<SensitiveDataSuggestionEvent[]> { return this.fail(); }
async listRelationships(): Promise<CatalogRelationship[]> { return this.fail(); }
async listRelationships(): Promise<CatalogPhysicalRelationship[]> { return this.fail(); }
async listLogicalRelationships(): Promise<CatalogLogicalRelationship[]> { return this.fail(); }
async getLogicalRelationshipContext(): Promise<CatalogLogicalRelationshipContext | undefined> { return this.fail(); }
async insertLogicalRelationship(): Promise<CatalogLogicalRelationship | undefined> { return this.fail(); }
async insertGeneratedLogicalRelationships(): Promise<number> { return this.fail(); }
async setLogicalRelationshipStatus(): Promise<CatalogLogicalRelationship | undefined> { return this.fail(); }
async deleteLogicalRelationship(): Promise<boolean> { return this.fail(); }
async deleteDatabaseMetadata(): Promise<CatalogMetadataDeleteCounts | undefined> { return this.fail(); }
async deleteTableMetadata(): Promise<CatalogMetadataDeleteCounts | undefined> { return this.fail(); }
async planSchemaSync(): Promise<CatalogSchemaDiff> { return this.fail(); }
+65 -2
View File
@@ -2,6 +2,7 @@ import { buildInstallationContract } from "../workspaces/contracts.js";
import { resolveBinding } from "../workspaces/bindings.js";
import type { WorkspaceRegistry } from "../workspaces/registry.js";
import type { WorkspaceDescriptor } from "../workspaces/schema.js";
import { discoverWorkspaceSecretRequirements } from "../workspaces/secret-requirements.js";
import type { WorkspaceSecretStore } from "../workspaces/secret-store.js";
import { createConcreteDiagnosticAdapters } from "../workspaces/diagnostics.js";
import { CatalogOperationCoordinator } from "./operation-coordinator.js";
@@ -25,6 +26,21 @@ export interface CatalogListItem extends Omit<WorkspaceDatabase, "id"> {
workspaceName: string;
workspaceDescription?: string;
workspaceAvailable: boolean;
workspaceRevision: { commit: string; blob: string } | null;
workspaceEvidence: {
sourceType: "filesystem" | "http" | "s3" | null;
state:
| "not_declared"
| "materialized_current_revision"
| "configuration_required"
| "configured_unverified"
| "workspace_unavailable";
};
runtimeBinding: {
transport: DatabaseBinding["transport"];
configurationState: "ready" | "configuration_required";
sessionTransportSupported: boolean;
} | null;
configured: boolean;
secrets: Record<CatalogSecretName, boolean>;
}
@@ -68,6 +84,43 @@ function secretState(store: WorkspaceSecretStore, workspaceId: string): Record<C
))) as Record<CatalogSecretName, boolean>;
}
function workspaceRuntimeState(
store: WorkspaceSecretStore,
workspace: WorkspaceDescriptor,
secretRoots: readonly string[],
): NonNullable<CatalogListItem["runtimeBinding"]> {
const requirements = discoverWorkspaceSecretRequirements(workspace, process.env);
const secretVariables = new Set(requirements.map(({ variable }) => variable));
const effective = resolveBinding(workspace, "DWH", process.env, secretRoots);
const configurationRequired = requirements.some(({ id, required }) => (
required && !store.has(workspace.workspace.id, id)
)) || effective.missing.some((variable) => !secretVariables.has(variable));
return {
transport: effective.transport,
configurationState: configurationRequired ? "configuration_required" : "ready",
sessionTransportSupported: effective.transport !== "ssh_tunnel",
};
}
function workspaceEvidenceState(
store: WorkspaceSecretStore,
workspace: WorkspaceDescriptor,
): CatalogListItem["workspaceEvidence"] {
const source = workspace.evidence?.source;
if (!source) return { sourceType: null, state: "not_declared" };
if (source.type === "filesystem") {
return { sourceType: source.type, state: "materialized_current_revision" };
}
const configurationRequired = discoverWorkspaceSecretRequirements(workspace, process.env)
.some(({ connector, id, required }) => (
connector === "evidence" && required && !store.has(workspace.workspace.id, id)
));
return {
sourceType: source.type,
state: configurationRequired ? "configuration_required" : "configured_unverified",
};
}
export class CatalogService {
private readonly adapters = createConcreteDiagnosticAdapters();
@@ -91,7 +144,8 @@ export class CatalogService {
const byWorkspace = new Map(configured.map((database) => [database.workspaceId, database]));
const active = await Promise.all(workspaces.map(async (entry) => {
const database = byWorkspace.get(entry.id);
const { workspace } = await this.registry.read(entry.id);
const { workspace } = await this.registry.readPinned(entry.id, entry.revision.commit);
const runtimeDatabaseBinding = yamlBinding(workspace, this.secretRoots);
const base = database ?? {
workspaceId: entry.id,
engine: "postgres" as const,
@@ -100,7 +154,7 @@ export class CatalogService {
version: 0,
createdAt: "",
updatedAt: "",
binding: yamlBinding(workspace, this.secretRoots),
binding: runtimeDatabaseBinding,
connectionStatus: "untested" as const,
};
return {
@@ -108,6 +162,12 @@ export class CatalogService {
workspaceName: entry.name,
workspaceDescription: entry.description,
workspaceAvailable: true,
workspaceRevision: {
commit: entry.revision.commit,
blob: entry.revision.blob,
},
workspaceEvidence: workspaceEvidenceState(this.secretStore, workspace),
runtimeBinding: workspaceRuntimeState(this.secretStore, workspace, this.secretRoots),
configured: database !== undefined,
secrets: secretState(this.secretStore, entry.id),
};
@@ -120,6 +180,9 @@ export class CatalogService {
workspaceName: database.workspaceId,
workspaceDescription: "Workspace is no longer present in the repository catalog.",
workspaceAvailable: false,
workspaceRevision: null,
workspaceEvidence: { sourceType: null, state: "workspace_unavailable" },
runtimeBinding: null,
configured: true,
secrets: secretState(this.secretStore, database.workspaceId),
}));
+66 -2
View File
@@ -128,7 +128,7 @@ export interface CatalogRelationshipColumn {
targetColumnName: string;
}
export interface CatalogRelationship {
export interface CatalogPhysicalRelationship {
id: string;
databaseId: string;
constraintName: string;
@@ -145,6 +145,52 @@ export interface CatalogRelationship {
lastSyncedAt: string | null;
createdAt: string;
updatedAt: string;
origin: "physical";
status: "active";
}
export interface CatalogLogicalRelationship {
id: string;
databaseId: string;
constraintName: null;
sourceTableId: string;
sourceTableName: string;
targetTableId: string;
targetTableName: string;
updateRule: null;
deleteRule: null;
deferrable: false;
initiallyDeferred: false;
columns: [CatalogRelationshipColumn];
lastSyncedDatabaseVersion: null;
lastSyncedAt: null;
createdAt: string;
updatedAt: string;
origin: "generated" | "manual";
status: "active" | "excluded";
}
export type CatalogRelationship = CatalogPhysicalRelationship | CatalogLogicalRelationship;
export interface CatalogLogicalRelationshipEndpoint {
columnId: string;
columnName: string;
tableId: string;
tableName: string;
dataType: string;
primaryKeyPosition: number | null;
tablePrimaryKeyColumnCount: number;
}
export interface CatalogLogicalRelationshipContext {
endpoints: CatalogLogicalRelationshipEndpoint[];
physicalPairs: Array<{ sourceColumnId: string; targetColumnId: string }>;
logicalRelationships: CatalogLogicalRelationship[];
}
export interface CatalogLogicalRelationshipCandidate {
sourceColumnId: string;
targetColumnId: string;
}
export type CatalogDatabaseMetadataDeleteTarget = "tables" | "relationships";
@@ -456,7 +502,25 @@ export interface CatalogRepository {
runId: string,
afterSequence?: number,
): Promise<SensitiveDataSuggestionEvent[]>;
listRelationships(databaseId: string): Promise<CatalogRelationship[]>;
listRelationships(databaseId: string): Promise<CatalogPhysicalRelationship[]>;
listLogicalRelationships(databaseId: string): Promise<CatalogLogicalRelationship[]>;
getLogicalRelationshipContext(databaseId: string): Promise<CatalogLogicalRelationshipContext | undefined>;
insertLogicalRelationship(
databaseId: string,
sourceColumnId: string,
targetColumnId: string,
generated: boolean,
): Promise<CatalogLogicalRelationship | undefined>;
insertGeneratedLogicalRelationships(
databaseId: string,
candidates: readonly CatalogLogicalRelationshipCandidate[],
): Promise<number>;
setLogicalRelationshipStatus(
databaseId: string,
relationshipId: string,
status: CatalogLogicalRelationship["status"],
): Promise<CatalogLogicalRelationship | undefined>;
deleteLogicalRelationship(databaseId: string, relationshipId: string): Promise<boolean>;
deleteDatabaseMetadata(
databaseIds: readonly string[],
target: CatalogDatabaseMetadataDeleteTarget,
@@ -0,0 +1,154 @@
import type { FastifyInstance, FastifyReply, FastifyRequest } from "fastify";
import { z } from "zod";
import { isPrincipalContext, requirePermission } from "../auth/authorization.js";
import {
CatalogLogicalRelationshipService,
LogicalRelationshipColumnNotFoundError,
LogicalRelationshipDatabaseNotFoundError,
LogicalRelationshipDuplicateError,
LogicalRelationshipNotFoundError,
LogicalRelationshipReadOnlyError,
LogicalRelationshipSchemaStaleError,
LogicalRelationshipTargetNotUniqueError,
LogicalRelationshipTypeIncompatibleError,
} from "../catalog/logical-relationship-service.js";
import type { CatalogOperationCoordinator } from "../catalog/operation-coordinator.js";
import { CatalogOperationInProgressError, CatalogUnavailableError } from "../catalog/types.js";
const idSchema = z.uuid();
const createSchema = z.object({
sourceColumnId: idSchema,
targetColumnId: idSchema,
}).strict();
const statusSchema = z.object({ status: z.enum(["active", "excluded"]) }).strict();
const emptySchema = z.object({}).strict();
function manage(request: FastifyRequest, reply: FastifyReply) {
return isPrincipalContext(requirePermission(request, reply, "database.manage"));
}
function safeError(reply: FastifyReply, error: unknown) {
if (error instanceof CatalogUnavailableError) {
return reply.code(503).send({ code: "catalog_unavailable", message: "Database catalog is unavailable." });
}
if (error instanceof CatalogOperationInProgressError) {
return reply.code(409).send({
code: "database_operation_in_progress",
message: "A database operation is already in progress.",
});
}
if (error instanceof LogicalRelationshipDatabaseNotFoundError) {
return reply.code(404).send({ code: "database_not_found", message: "Database configuration was not found." });
}
if (error instanceof LogicalRelationshipColumnNotFoundError) {
return reply.code(404).send({
code: "column_not_found",
message: "Catalog column was not found.",
field: error.field,
});
}
if (error instanceof LogicalRelationshipNotFoundError) {
return reply.code(404).send({ code: "relationship_not_found", message: "Relationship was not found." });
}
if (error instanceof LogicalRelationshipDuplicateError) {
return reply.code(409).send({ code: "relationship_duplicate", message: "Relationship already exists." });
}
if (error instanceof LogicalRelationshipReadOnlyError) {
return reply.code(409).send({ code: "relationship_read_only", message: "Physical relationships are read-only." });
}
if (error instanceof LogicalRelationshipSchemaStaleError) {
return reply.code(409).send({
code: "relationship_schema_stale",
message: "Synchronize the current database schema before managing logical relationships.",
});
}
if (error instanceof LogicalRelationshipTargetNotUniqueError) {
return reply.code(422).send({
code: "relationship_target_not_unique",
message: "Target column must be the only primary-key column of its table.",
field: "targetColumnId",
});
}
if (error instanceof LogicalRelationshipTypeIncompatibleError) {
return reply.code(422).send({
code: "relationship_type_incompatible",
message: "Source and target column types are incompatible.",
field: "targetColumnId",
});
}
if (error instanceof z.ZodError) {
return reply.code(400).send({
code: "relationship_request_invalid",
message: "Relationship request is invalid.",
});
}
return reply.code(500).send({
code: "relationship_operation_failed",
message: "Relationship operation failed.",
});
}
export function catalogLogicalRelationshipRoutes(
app: FastifyInstance,
deps: {
service: CatalogLogicalRelationshipService;
operations: CatalogOperationCoordinator;
},
): void {
app.get("/catalog/databases/:databaseId/relationships", async (request, reply) => {
if (!manage(request, reply)) return reply;
try {
const databaseId = idSchema.parse((request.params as { databaseId?: unknown }).databaseId);
return await deps.service.list(databaseId);
} catch (error) { return safeError(reply, error); }
});
app.post("/catalog/databases/:databaseId/relationships", async (request, reply) => {
if (!manage(request, reply)) return reply;
try {
const databaseId = idSchema.parse((request.params as { databaseId?: unknown }).databaseId);
const input = createSchema.parse(request.body);
const relationship = await deps.operations.run(databaseId, async () => (
await deps.service.addManual(databaseId, input.sourceColumnId, input.targetColumnId)
));
return reply.code(201).send(relationship);
} catch (error) { return safeError(reply, error); }
});
app.post("/catalog/databases/:databaseId/relationships/rebuild-generated", async (request, reply) => {
if (!manage(request, reply)) return reply;
try {
const databaseId = idSchema.parse((request.params as { databaseId?: unknown }).databaseId);
emptySchema.parse(request.body ?? {});
return await deps.operations.run(databaseId, async () => (
await deps.service.rebuildGenerated(databaseId)
));
} catch (error) { return safeError(reply, error); }
});
app.patch("/catalog/databases/:databaseId/relationships/:relationshipId", async (request, reply) => {
if (!manage(request, reply)) return reply;
try {
const params = request.params as { databaseId?: unknown; relationshipId?: unknown };
const databaseId = idSchema.parse(params.databaseId);
const relationshipId = idSchema.parse(params.relationshipId);
const input = statusSchema.parse(request.body);
return await deps.operations.run(databaseId, async () => (
await deps.service.setStatus(databaseId, relationshipId, input.status)
));
} catch (error) { return safeError(reply, error); }
});
app.delete("/catalog/databases/:databaseId/relationships/:relationshipId", async (request, reply) => {
if (!manage(request, reply)) return reply;
try {
const params = request.params as { databaseId?: unknown; relationshipId?: unknown };
const databaseId = idSchema.parse(params.databaseId);
const relationshipId = idSchema.parse(params.relationshipId);
await deps.operations.run(databaseId, async () => {
await deps.service.deletePermanently(databaseId, relationshipId);
});
return reply.code(204).send();
} catch (error) { return safeError(reply, error); }
});
}
-11
View File
@@ -131,17 +131,6 @@ export function catalogSchemaRoutes(
} catch (error) { return safeError(reply, error); }
});
app.get("/catalog/databases/:databaseId/relationships", async (request, reply) => {
if (!manage(request, reply)) return reply;
try {
const databaseId = idSchema.parse((request.params as { databaseId?: unknown }).databaseId);
if (!(await deps.repository.get(databaseId))) {
return reply.code(404).send({ code: "database_not_found", message: "Database configuration was not found." });
}
return await deps.repository.listRelationships(databaseId);
} catch (error) { return safeError(reply, error); }
});
app.post("/catalog/databases/metadata-cleanup", async (request, reply) => {
if (!manage(request, reply)) return reply;
try {
+35 -8
View File
@@ -11,6 +11,7 @@ import type { WorkspaceRegistry } from "../workspaces/registry.js";
import { validateOperationalWorkspace, type WorkspaceDescriptor } from "../workspaces/schema.js";
import type { MaintenanceBarrier } from "../runtime/maintenance-gate.js";
import { hasPermission, isPrincipalContext, requirePermission } from "../auth/authorization.js";
import type { EffectiveRelationshipSnapshotProvider } from "../catalog/effective-relationship-snapshot.js";
const BOOTSTRAP_FAILURE_MESSAGE =
"Session startup failed. Check configuration and connectivity, then Resume the session.";
@@ -40,6 +41,8 @@ export function sessionRoutes(
/** Fail-closed installation/runtime transport capability check. */
workspaceRuntimeSupport: (workspace: WorkspaceDescriptor) => boolean;
maintenanceBarrier: MaintenanceBarrier;
/** Optional only for narrow route-test stubs and installations without a Catalog database. */
effectiveRelationships?: EffectiveRelationshipSnapshotProvider;
},
) {
const lifecycleTails = new Map<string, Promise<void>>();
@@ -84,11 +87,21 @@ export function sessionRoutes(
isAdmin: hasPermission(principal, permission),
});
const optionsWithRuntimeConfig = (runner: any, workspaceConfigPath: string | undefined, options: any) => (
workspaceConfigPath && typeof runner.acquireWorkspaceRuntime === "function"
? { ...options, runtimeConfig: runner.acquireWorkspaceRuntime(workspaceConfigPath) }
: options
);
const optionsWithRuntimeConfig = async (
runner: any,
workspaceConfigPath: string | undefined,
workspaceId: string | undefined,
options: any,
) => {
if (!workspaceConfigPath || typeof runner.acquireWorkspaceRuntime !== "function") return options;
const effectiveRelationships = workspaceId && d.effectiveRelationships
? await d.effectiveRelationships.render(workspaceId)
: undefined;
return {
...options,
runtimeConfig: runner.acquireWorkspaceRuntime(workspaceConfigPath, effectiveRelationships),
};
};
const maintenanceReply = (reply: any) => reply.code(503).send({
code: "maintenance",
@@ -448,7 +461,12 @@ export function sessionRoutes(
let runtimeOptions = options;
let rt: ReturnType<PiProcessManager["createFor"]> | undefined;
try {
runtimeOptions = optionsWithRuntimeConfig(runner, workspaceConfigPath, options);
runtimeOptions = await optionsWithRuntimeConfig(
runner,
workspaceConfigPath,
workspaceId,
options,
);
rt = d.mgr.createFor(id, runtimeOptions);
bindRuntime(id, rt, runner, workspaceConfigPath);
} catch (error) {
@@ -465,7 +483,11 @@ export function sessionRoutes(
info(id, "Session created");
bootstrap(
id, rt, runner, workspaceConfigPath, d.mgr.configure(rt, runtimeOptions),
runner.searchPack(b.question, id, workspaceConfigPath),
runner.searchPack(
b.question,
id,
(runtimeOptions as any).runtimeConfig?.path ?? workspaceConfigPath,
),
() => d.mgr.start(id, rt, runtimeOptions),
);
return { id };
@@ -639,7 +661,12 @@ export function sessionRoutes(
if (boundRuntimes.get(id) === current) boundRuntimes.delete(id);
d.mgr.teardownIfCurrent(id, current);
}
runtimeOptions = optionsWithRuntimeConfig(runner, workspaceConfigPath, options);
runtimeOptions = await optionsWithRuntimeConfig(
runner,
workspaceConfigPath,
saved.workspace_id,
options,
);
rt = d.mgr.createFor(id, runtimeOptions);
bindRuntime(id, rt, runner, workspaceConfigPath);
} catch {
+27 -12
View File
@@ -213,23 +213,37 @@ export class ThtRunner {
}
/** Render one immutable canonical registry revision into a backend-owned harness config. */
acquireWorkspaceRuntime(workspaceConfigPath: string): RuntimeConfigLease {
const rendered = renderWorkspaceRuntimeFromSnapshotPath({
snapshotPath: workspaceConfigPath,
harnessDir: this.cfg.harnessDir,
configPath: this.cfg.configPath,
dataRoot: this.cfg.dataRoot ?? (() => {
throw new Error("registry workspace runtime requires an absolute data root");
})(),
secretRoots: this.cfg.secretRoots ?? [],
semanticRuntime: this.cfg.semanticRuntime ?? DEFAULT_SEMANTIC_RUNTIME,
workspaceSecretStore: this.cfg.workspaceSecretStore,
});
acquireWorkspaceRuntime(
workspaceConfigPath: string,
effectiveRelationships?: string,
): RuntimeConfigLease {
const effectiveRelationshipsPath = effectiveRelationships === undefined
? undefined
: this.createRuntimeSnapshot(effectiveRelationships);
let rendered: ReturnType<typeof renderWorkspaceRuntimeFromSnapshotPath>;
try {
rendered = renderWorkspaceRuntimeFromSnapshotPath({
snapshotPath: workspaceConfigPath,
harnessDir: this.cfg.harnessDir,
configPath: this.cfg.configPath,
dataRoot: this.cfg.dataRoot ?? (() => {
throw new Error("registry workspace runtime requires an absolute data root");
})(),
secretRoots: this.cfg.secretRoots ?? [],
semanticRuntime: this.cfg.semanticRuntime ?? DEFAULT_SEMANTIC_RUNTIME,
workspaceSecretStore: this.cfg.workspaceSecretStore,
effectiveRelationshipsPath,
});
} catch (error) {
if (effectiveRelationshipsPath) this.cleanupRuntimeSnapshot(effectiveRelationshipsPath);
throw error;
}
let path: string;
try {
path = this.createRuntimeSnapshot(rendered.renderedConfig);
} catch (error) {
rendered.releaseSecrets();
if (effectiveRelationshipsPath) this.cleanupRuntimeSnapshot(effectiveRelationshipsPath);
throw error;
}
let released = false;
@@ -241,6 +255,7 @@ export class ThtRunner {
if (released) return;
released = true;
this.cleanupRuntimeSnapshot(path);
if (effectiveRelationshipsPath) this.cleanupRuntimeSnapshot(effectiveRelationshipsPath);
rendered.releaseSecrets();
},
};
+24 -3
View File
@@ -243,7 +243,12 @@ function readSnapshotWorkspace(snapshotPath: string): {
}
}
function runtimePaths(dataRoot: string, workspaceId: string, workspaceRevision?: string): RuntimePaths {
function runtimePaths(
dataRoot: string,
workspaceId: string,
workspaceRevision?: string,
effectiveRelationshipsPath?: string,
): RuntimePaths {
if (!isAbsolute(dataRoot)) throw new Error("registry workspace runtime requires an absolute data root");
const root = join(dataRoot, "sessions", workspaceId);
return {
@@ -254,6 +259,9 @@ function runtimePaths(dataRoot: string, workspaceId: string, workspaceRevision?:
...(workspaceRevision === undefined
? {}
: { annotations_root: join(dataRoot, "sessions", workspaceId, "revisions", workspaceRevision, "artifacts") }),
...(effectiveRelationshipsPath === undefined
? {}
: { effective_relationships: effectiveRelationshipsPath }),
};
}
@@ -301,6 +309,7 @@ function renderWorkspaceRuntimeFromWorkspace(options: {
dataRoot: string;
secretRoots: readonly string[];
semanticRuntime: SemanticRuntimeConfig;
effectiveRelationshipsPath?: string;
workspaceSecretStore?: WorkspaceSecretStore;
}): RenderedWorkspaceRuntime {
const secretLease = options.workspaceSecretStore === undefined
@@ -325,7 +334,12 @@ function renderWorkspaceRuntimeFromWorkspace(options: {
workspaceId: options.workspaceId,
workspaceRevision: options.workspaceRevision,
revisionContentRoot: options.revisionContentRoot,
runtimePaths: runtimePaths(options.dataRoot, options.workspaceId, options.workspaceRevision),
runtimePaths: runtimePaths(
options.dataRoot,
options.workspaceId,
options.workspaceRevision,
options.effectiveRelationshipsPath,
),
installationOverlay: overlay,
bindings,
bindingDigest: stableBindingDigest(bindings),
@@ -334,7 +348,12 @@ function renderWorkspaceRuntimeFromWorkspace(options: {
renderedConfig: renderRuntimeConfig(
options.workspace,
bindings,
runtimePaths(options.dataRoot, options.workspaceId, options.workspaceRevision),
runtimePaths(
options.dataRoot,
options.workspaceId,
options.workspaceRevision,
options.effectiveRelationshipsPath,
),
context,
overlay,
options.semanticRuntime,
@@ -353,6 +372,7 @@ export function renderWorkspaceRuntimeFromSnapshotPath(options: {
dataRoot: string;
secretRoots: readonly string[];
semanticRuntime: SemanticRuntimeConfig;
effectiveRelationshipsPath?: string;
workspaceSecretStore?: WorkspaceSecretStore;
}): RenderedWorkspaceRuntime {
const snapshot = readSnapshotWorkspace(options.snapshotPath);
@@ -366,6 +386,7 @@ export function renderWorkspaceRuntimeFromSnapshotPath(options: {
dataRoot: options.dataRoot,
secretRoots: options.secretRoots,
semanticRuntime: options.semanticRuntime,
effectiveRelationshipsPath: options.effectiveRelationshipsPath,
workspaceSecretStore: options.workspaceSecretStore,
});
}
@@ -12,6 +12,8 @@ export interface RuntimePaths {
memory: string;
/** Revision-qualified root for curated FK annotations (P5); optional for legacy callers. */
annotations_root?: string;
/** Immutable Catalog projection used as the exclusive runtime relationship source. */
effective_relationships?: string;
}
export interface RuntimeIdentity {
+63 -4
View File
@@ -13,7 +13,10 @@ import type { WorkspaceRegistry, WorkspaceRevision } from "../src/workspaces/reg
import type { WorkspaceDescriptor } from "../src/workspaces/schema.js";
const roots: string[] = [];
afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); });
afterEach(() => {
vi.unstubAllEnvs();
for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true });
});
const workspace: WorkspaceDescriptor = {
workspace: { schema_version: 3, id: "psd-clinical", name: "Policlinico San Donato", language: "it" },
@@ -36,6 +39,7 @@ function setup(
catalogOperationCoordinator?: CatalogOperationCoordinator;
catalogPostgresAccess?: CatalogPostgresAccess;
} = {},
workspaceDescriptor: WorkspaceDescriptor = workspace,
) {
const secretRoot = mkdtempSync(join(tmpdir(), "catalog-secret-"));
const runtimeRoot = mkdtempSync(join(tmpdir(), "catalog-secret-runtime-"));
@@ -45,7 +49,8 @@ function setup(
const registry = {
list: vi.fn(async () => [revision]),
listCatalog: vi.fn(async () => [{ id: "psd-clinical", name: "Policlinico San Donato", configurationState: "ready", revision }]),
read: vi.fn(async () => ({ workspace, revision })),
read: vi.fn(async () => ({ workspace: workspaceDescriptor, revision })),
readPinned: vi.fn(async () => ({ workspace: workspaceDescriptor, workspaceConfigPath: revision.snapshotPath })),
} as unknown as WorkspaceRegistry;
const app = buildApp(loadConfig({
THT_HARNESS_DIR: "/missing",
@@ -96,10 +101,31 @@ const fleetSnapshot: ObservedSchemaSnapshot = {
};
test("lists every YAML workspace and creates its one database configuration", async () => {
const { app } = setup();
const { app, secretStore } = setup();
const initial = await app.inject({ method: "GET", url: "/catalog/databases" });
expect(initial.statusCode).toBe(200);
expect(initial.json()).toMatchObject([{ workspaceId: "psd-clinical", configured: false, databaseName: "warehouse" }]);
expect(initial.json()).toMatchObject([{
workspaceId: "psd-clinical",
configured: false,
databaseName: "warehouse",
workspaceRevision: { commit: revision.commit, blob: revision.blob },
workspaceEvidence: { sourceType: null, state: "not_declared" },
runtimeBinding: {
transport: "postgres_direct",
configurationState: "configuration_required",
sessionTransportSupported: true,
},
}]);
vi.stubEnv("THT_WS_PSD_CLINICAL_DWH_HOST", "runtime-db.internal");
vi.stubEnv("THT_WS_PSD_CLINICAL_DWH_PORT", "5432");
vi.stubEnv("THT_WS_PSD_CLINICAL_DWH_USER", "runtime-reader");
vi.stubEnv("THT_WS_PSD_CLINICAL_DWH_TRANSPORT", "postgres_direct");
secretStore.putMany("psd-clinical", { "dwh.password": "runtime-password" });
const runtimeReady = await app.inject({ method: "GET", url: "/catalog/databases" });
expect(runtimeReady.json()).toMatchObject([{
runtimeBinding: { configurationState: "ready", sessionTransportSupported: true },
}]);
const created = await app.inject({ method: "POST", url: "/catalog/databases", payload: direct });
expect(created.statusCode).toBe(201);
@@ -110,6 +136,39 @@ test("lists every YAML workspace and creates its one database configuration", as
expect(listed.json()).toMatchObject([{ configured: true, binding: { transport: "postgres_direct", host: "db.internal" } }]);
});
test("projects remote Evidence credential state without conflating catalog secrets", async () => {
const evidenceWorkspace: WorkspaceDescriptor = {
...workspace,
evidence: {
schema_version: 2,
source: {
type: "http",
uris: ["https://evidence.example.test/guide.md"],
authentication: "signed_urls_file",
connect_timeout_ms: 5_000,
read_timeout_ms: 30_000,
max_bytes: 10 * 1024 * 1024,
max_redirects: 5,
allow_private_hosts: false,
max_cache_bytes: 64 * 1024 * 1024,
},
policy: { max_chunk_chars: 4_000, retain_published_generations: 3 },
},
};
const { app, secretStore } = setup({}, {}, evidenceWorkspace);
const missing = await app.inject({ method: "GET", url: "/catalog/databases" });
expect(missing.json()).toMatchObject([{
workspaceEvidence: { sourceType: "http", state: "configuration_required" },
}]);
secretStore.putMany("psd-clinical", { "evidence.signed_urls": "https://signed.example.test/evidence" });
const configured = await app.inject({ method: "GET", url: "/catalog/databases" });
expect(configured.json()).toMatchObject([{
workspaceEvidence: { sourceType: "http", state: "configured_unverified" },
}]);
});
test("lists orphaned records and takes the REST diagnostic path from workspace YAML", async () => {
const { app, repository } = setup();
await repository.create({
@@ -0,0 +1,207 @@
import { expect, test } from "vitest";
import {
CatalogLogicalRelationshipService,
LogicalRelationshipDuplicateError,
LogicalRelationshipSchemaStaleError,
LogicalRelationshipTargetNotUniqueError,
LogicalRelationshipTypeIncompatibleError,
} from "../src/catalog/logical-relationship-service.js";
import { MemoryCatalogRepository } from "../src/catalog/memory-repository.js";
import type { ObservedSchemaSnapshot } from "../src/catalog/types.js";
const column = (
tableName: string,
name: string,
ordinalPosition: number,
dataType: string,
primaryKeyPosition: number | null,
) => ({
tableName, name, ordinalPosition, dataType, primaryKeyPosition,
isNullable: false, defaultExpression: null, sourceComment: null,
});
function schema(
tableNames: string[],
columns: ObservedSchemaSnapshot["columns"],
relationships: ObservedSchemaSnapshot["relationships"] = [],
): ObservedSchemaSnapshot {
return {
schemaVersion: 1,
capabilities: { tables: "available", columns: "available", relationships: "available" },
tables: tableNames.map((name) => ({ name, sourceComment: null })),
columns,
relationships,
};
}
async function setup(snapshot: ObservedSchemaSnapshot) {
const repository = new MemoryCatalogRepository();
const database = await repository.create({
workspaceId: "relationships", engine: "postgres", databaseName: "warehouse", schema: "public",
binding: { transport: "postgres_direct", host: "db.internal", port: 5432, username: "reader" },
});
await repository.applySchemaSync(database.id, database.version, "all", [], snapshot);
const service = new CatalogLogicalRelationshipService(repository);
const context = (await repository.getLogicalRelationshipContext(database.id))!;
const endpoint = (tableName: string, columnName: string) => context.endpoints.find((item) => (
item.tableName === tableName && item.columnName === columnName
))!;
return { repository, database, service, endpoint };
}
test("keeps excluded relationships across rebuild and recreates hard-deleted relationships", async () => {
const { database, service, endpoint } = await setup(schema(
["users", "orders"],
[column("users", "id", 1, "bigint", 1), column("orders", "id", 1, "bigint", 1),
column("orders", "user_id", 2, "bigint", null)],
));
const source = endpoint("orders", "user_id");
const target = endpoint("users", "id");
const manual = await service.addManual(database.id, source.columnId, target.columnId);
expect(manual).toMatchObject({ origin: "manual", status: "active" });
expect(await service.setStatus(database.id, manual.id, "excluded"))
.toMatchObject({ status: "excluded" });
await expect(service.rebuildGenerated(database.id)).resolves.toEqual({
added: 0, alreadyPresent: 0, excluded: 1, ambiguous: 0,
});
expect((await service.list(database.id)).filter((item) => item.origin !== "physical"))
.toMatchObject([{ id: manual.id, origin: "manual", status: "excluded" }]);
await service.deletePermanently(database.id, manual.id);
await expect(service.rebuildGenerated(database.id)).resolves.toEqual({
added: 1, alreadyPresent: 0, excluded: 0, ambiguous: 0,
});
expect((await service.list(database.id)).filter((item) => item.origin !== "physical"))
.toMatchObject([{ origin: "generated", status: "active" }]);
});
test("normalizes snake, kebab, and camel names without fuzzy matching", async () => {
const { database, service } = await setup(schema(
["users", "events"],
[column("users", "id", 1, "bigint", 1), column("events", "id", 1, "bigint", 1),
column("events", "user_id", 2, "bigint", null),
column("events", "user-id", 3, "bigint", null),
column("events", "userId", 4, "bigint", null),
column("events", "userid", 5, "bigint", null),
column("events", "unrelated", 6, "bigint", null)],
));
await expect(service.rebuildGenerated(database.id)).resolves.toEqual({
added: 4, alreadyPresent: 0, excluded: 0, ambiguous: 0,
});
});
test("skips generic bare primary-key names while retaining table-qualified matches", async () => {
const { database, service } = await setup(schema(
["users", "accounts", "events"],
[column("users", "id", 1, "bigint", 1), column("accounts", "id", 1, "bigint", 1),
column("events", "event_key", 1, "bigint", 1), column("events", "id", 2, "bigint", null),
column("events", "user_id", 3, "bigint", null)],
));
await expect(service.rebuildGenerated(database.id)).resolves.toEqual({
added: 1, alreadyPresent: 0, excluded: 0, ambiguous: 0,
});
});
test("infers foreign keys from composite-primary-key sources", async () => {
const { database, service } = await setup(schema(
["users", "groups", "memberships"],
[column("users", "id", 1, "bigint", 1), column("groups", "id", 1, "bigint", 1),
column("memberships", "user_id", 1, "bigint", 1),
column("memberships", "group_id", 2, "bigint", 2)],
));
await expect(service.rebuildGenerated(database.id)).resolves.toEqual({
added: 2, alreadyPresent: 0, excluded: 0, ambiguous: 0,
});
});
test("maps time-key columns to the single primary key of dim_time", async () => {
const { database, service } = await setup(schema(
["dim_time", "admissions"],
[column("dim_time", "day_key", 1, "integer", 1),
column("admissions", "id", 1, "bigint", 1),
column("admissions", "admission_time_key", 2, "integer", null),
column("admissions", "discharge_time_key", 3, "integer", null)],
));
await expect(service.rebuildGenerated(database.id)).resolves.toEqual({
added: 2, alreadyPresent: 0, excluded: 0, ambiguous: 0,
});
});
test("skips ambiguous targets and physical foreign-key pairs", async () => {
const ambiguous = await setup(schema(
["user", "users", "events"],
[column("user", "id", 1, "bigint", 1), column("users", "id", 1, "bigint", 1),
column("events", "id", 1, "bigint", 1), column("events", "user_id", 2, "bigint", null)],
));
await expect(ambiguous.service.rebuildGenerated(ambiguous.database.id)).resolves.toEqual({
added: 0, alreadyPresent: 0, excluded: 0, ambiguous: 1,
});
const physical = await setup(schema(
["users", "orders"],
[column("users", "id", 1, "bigint", 1), column("orders", "id", 1, "bigint", 1),
column("orders", "user_id", 2, "bigint", null)],
[{
constraintName: "orders_user_id_fkey", sourceTableName: "orders", targetTableName: "users",
updateRule: "NO ACTION", deleteRule: "NO ACTION", deferrable: false, initiallyDeferred: false,
columns: [{ position: 1, sourceColumnName: "user_id", targetColumnName: "id" }],
}],
));
await expect(physical.service.rebuildGenerated(physical.database.id)).resolves.toEqual({
added: 0, alreadyPresent: 1, excluded: 0, ambiguous: 0,
});
await expect(physical.service.addManual(
physical.database.id,
physical.endpoint("orders", "user_id").columnId,
physical.endpoint("users", "id").columnId,
)).rejects.toBeInstanceOf(LogicalRelationshipDuplicateError);
});
test("validates target uniqueness and canonical type compatibility for manual relationships", async () => {
const { database, service, endpoint } = await setup(schema(
["users", "composite", "events"],
[column("users", "id", 1, "bigint", 1),
column("composite", "left_id", 1, "bigint", 1), column("composite", "right_id", 2, "bigint", 2),
column("events", "id", 1, "bigint", 1), column("events", "user_id", 2, "integer", null),
column("events", "composite_id", 3, "bigint", null)],
));
await expect(service.addManual(
database.id, endpoint("events", "composite_id").columnId, endpoint("composite", "left_id").columnId,
)).rejects.toBeInstanceOf(LogicalRelationshipTargetNotUniqueError);
await expect(service.addManual(
database.id, endpoint("events", "user_id").columnId, endpoint("users", "id").columnId,
)).rejects.toBeInstanceOf(LogicalRelationshipTypeIncompatibleError);
});
test("rebuild is additive when an existing generated relationship stops matching", async () => {
const initial = schema(
["users", "orders"],
[column("users", "id", 1, "bigint", 1), column("orders", "id", 1, "bigint", 1),
column("orders", "user_id", 2, "bigint", null)],
);
const { repository, database, service } = await setup(initial);
await service.rebuildGenerated(database.id);
const changed = structuredClone(initial);
changed.columns.find((item) => item.tableName === "orders" && item.name === "user_id")!.dataType = "text";
await repository.applySchemaSync(database.id, database.version, "columns", [], changed);
await expect(service.rebuildGenerated(database.id)).resolves.toEqual({
added: 0, alreadyPresent: 0, excluded: 0, ambiguous: 0,
});
expect((await service.list(database.id)).filter((item) => item.origin === "generated")).toHaveLength(1);
});
test("refuses inference until the current database version has a full schema sync", async () => {
const repository = new MemoryCatalogRepository();
const database = await repository.create({
workspaceId: "unsynced-relationships",
engine: "postgres",
databaseName: "warehouse",
schema: "public",
binding: { transport: "postgres_direct", host: "db.internal", port: 5432, username: "reader" },
});
const service = new CatalogLogicalRelationshipService(repository);
await expect(service.rebuildGenerated(database.id))
.rejects.toBeInstanceOf(LogicalRelationshipSchemaStaleError);
});
@@ -12,6 +12,7 @@ import { up as upRuntimeSequencePrivileges } from "../src/catalog/migrations/004
import { up as upDescriptionGeneration } from "../src/catalog/migrations/005_description_generation_runs.js";
import { up as upSensitiveDataFlag } from "../src/catalog/migrations/006_sensitive_data_flag.js";
import { up as upSensitiveSuggestionRuns } from "../src/catalog/migrations/007_sensitive_data_suggestion_runs.js";
import { up as upLogicalRelationships } from "../src/catalog/migrations/008_catalog_logical_relationships.js";
const dockerAvailable = spawnSync("docker", ["info"], { stdio: "ignore" }).status === 0;
@@ -26,6 +27,7 @@ test.skipIf(!dockerAvailable)("PostgreSQL migration enforces one database per wo
await upTables(db);
await upSchemaSync(db);
await upSensitiveDataFlag(db);
await upLogicalRelationships(db);
await sql`CREATE ROLE thothii_catalog_runtime`.execute(db);
await upRuntimeSequencePrivileges(db);
const sequencePrivilege = await sql<{ allowed: boolean }>`
@@ -202,6 +204,7 @@ test.skipIf(!dockerAvailable)("PostgreSQL repository performs scoped metadata cl
await upTables(db);
await upSchemaSync(db);
await upSensitiveDataFlag(db);
await upLogicalRelationships(db);
const repository = new KyselyCatalogRepository(db);
const database = await repository.create({
workspaceId: "cleanup-test",
@@ -280,6 +283,7 @@ test.skipIf(!dockerAvailable)("PostgreSQL repository atomically consolidates sel
await upTables(db);
await upSchemaSync(db);
await upSensitiveDataFlag(db);
await upLogicalRelationships(db);
const repository = new KyselyCatalogRepository(db);
const database = await repository.create({
workspaceId: "consolidation-test",
@@ -379,6 +383,7 @@ test.skipIf(!dockerAvailable)("PostgreSQL repository persists description and se
await upTables(db);
await upSchemaSync(db);
await upSensitiveDataFlag(db);
await upLogicalRelationships(db);
await upDescriptionGeneration(db);
await upSensitiveSuggestionRuns(db);
const repository = new KyselyCatalogRepository(db);
@@ -610,3 +615,63 @@ test.skipIf(!dockerAvailable)("PostgreSQL repository persists description and se
await container.stop();
}
}, 60_000);
test.skipIf(!dockerAvailable)("PostgreSQL repository persists logical relationship lifecycle and tombstones", async () => {
const container = await new PostgreSqlContainer("postgres:17.6-bookworm").start();
const db = new Kysely<CatalogDatabase>({
dialect: new PostgresDialect({ pool: new Pool({ connectionString: container.getConnectionUri() }) }),
plugins: [new CamelCasePlugin()],
});
try {
await upDatabases(db);
await upTables(db);
await upSchemaSync(db);
await upSensitiveDataFlag(db);
await upLogicalRelationships(db);
const repository = new KyselyCatalogRepository(db);
const database = await repository.create({
workspaceId: "logical-relationships",
engine: "postgres",
databaseName: "warehouse",
schema: "public",
binding: { transport: "postgres_direct", host: "db.internal", port: 5432, username: "reader" },
});
await repository.applySchemaSync(database.id, database.version, "all", [], {
schemaVersion: 1,
capabilities: { tables: "available", columns: "available", relationships: "available" },
tables: [{ name: "users", sourceComment: null }, { name: "orders", sourceComment: null }],
columns: [
{ tableName: "users", name: "id", ordinalPosition: 1, dataType: "bigint", isNullable: false, defaultExpression: null, primaryKeyPosition: 1, sourceComment: null },
{ tableName: "orders", name: "id", ordinalPosition: 1, dataType: "bigint", isNullable: false, defaultExpression: null, primaryKeyPosition: 1, sourceComment: null },
{ tableName: "orders", name: "user_id", ordinalPosition: 2, dataType: "bigint", isNullable: false, defaultExpression: null, primaryKeyPosition: null, sourceComment: null },
],
relationships: [],
});
const context = (await repository.getLogicalRelationshipContext(database.id))!;
const source = context.endpoints.find((item) => item.tableName === "orders" && item.columnName === "user_id")!;
const target = context.endpoints.find((item) => item.tableName === "users" && item.columnName === "id")!;
const created = await repository.insertLogicalRelationship(database.id, source.columnId, target.columnId, false);
expect(created).toMatchObject({ origin: "manual", status: "active" });
await expect(repository.insertLogicalRelationship(database.id, source.columnId, target.columnId, true))
.resolves.toBeUndefined();
expect(await repository.setLogicalRelationshipStatus(database.id, created!.id, "excluded"))
.toMatchObject({ status: "excluded" });
await expect(repository.insertGeneratedLogicalRelationships(database.id, [{
sourceColumnId: source.columnId, targetColumnId: target.columnId,
}])).resolves.toBe(0);
await expect(repository.deleteLogicalRelationship(database.id, created!.id)).resolves.toBe(true);
await expect(repository.insertGeneratedLogicalRelationships(database.id, [{
sourceColumnId: source.columnId, targetColumnId: target.columnId,
}])).resolves.toBe(1);
expect(await repository.listLogicalRelationships(database.id))
.toMatchObject([{ origin: "generated", status: "active" }]);
await db.deleteFrom("catalogColumns").where("id", "=", source.columnId).execute();
expect(await repository.listLogicalRelationships(database.id)).toEqual([]);
} finally {
await db.destroy();
await container.stop();
}
}, 60_000);
+123
View File
@@ -122,6 +122,7 @@ async function setup(env: Record<string, string> = {}) {
list: vi.fn(async () => [revision]),
listCatalog: vi.fn(async () => [{ id: "psd-clinical", name: "Policlinico San Donato", configurationState: "ready", revision }]),
read: vi.fn(async () => ({ workspace, revision })),
readPinned: vi.fn(async () => ({ workspace, workspaceConfigPath: revision.snapshotPath })),
} as unknown as WorkspaceRegistry;
const app = buildApp(loadConfig({ THT_HARNESS_DIR: "/missing", NODE_ENV: "test", ...env }), {
thtRunner: {} as never,
@@ -605,3 +606,125 @@ test("rejects a missing database without partially cleaning valid selections", a
expect(response.statusCode).toBe(404);
expect(await repository.listTables(database.id)).toHaveLength(2);
});
test("serves one relationship map and supports the manual relationship lifecycle", async () => {
const { app, repository, database } = await setup();
await seedCatalog(repository, database);
const tables = await repository.listTables(database.id);
const patients = tables.find((table) => table.name === "patients")!;
const visits = tables.find((table) => table.name === "visits")!;
const patientId = (await repository.listColumns(database.id, patients.id)).find((item) => item.name === "id")!;
const visitId = (await repository.listColumns(database.id, visits.id)).find((item) => item.name === "id")!;
const created = await app.inject({
method: "POST",
url: `/catalog/databases/${database.id}/relationships`,
payload: { sourceColumnId: visitId.id, targetColumnId: patientId.id },
});
expect(created.statusCode).toBe(201);
expect(created.json()).toMatchObject({ origin: "manual", status: "active", constraintName: null });
const listed = (await app.inject({
method: "GET", url: `/catalog/databases/${database.id}/relationships`,
})).json();
expect(listed.map((item: { origin: string }) => item.origin).sort()).toEqual(["manual", "physical"]);
const relationshipId = created.json().id;
const excluded = await app.inject({
method: "PATCH",
url: `/catalog/databases/${database.id}/relationships/${relationshipId}`,
payload: { status: "excluded" },
});
expect(excluded.statusCode).toBe(200);
expect(excluded.json()).toMatchObject({ origin: "manual", status: "excluded" });
const restored = await app.inject({
method: "PATCH",
url: `/catalog/databases/${database.id}/relationships/${relationshipId}`,
payload: { status: "active" },
});
expect(restored.json()).toMatchObject({ status: "active" });
expect((await app.inject({
method: "DELETE", url: `/catalog/databases/${database.id}/relationships/${relationshipId}`,
})).statusCode).toBe(204);
});
test("rejects generated inference while the Catalog schema is not current", async () => {
const { app, database } = await setup();
const response = await app.inject({
method: "POST",
url: `/catalog/databases/${database.id}/relationships/rebuild-generated`,
});
expect(response.statusCode).toBe(409);
expect(response.json()).toEqual({
code: "relationship_schema_stale",
message: "Synchronize the current database schema before managing logical relationships.",
});
});
test("rebuilds generated relationships and returns the exact summary", async () => {
const { app, repository, database } = await setup();
const observed = snapshot();
observed.relationships = [];
await seedCatalog(repository, database, observed);
const first = await app.inject({
method: "POST", url: `/catalog/databases/${database.id}/relationships/rebuild-generated`,
});
expect(first.statusCode).toBe(200);
expect(first.json()).toEqual({ added: 1, alreadyPresent: 0, excluded: 0, ambiguous: 0 });
const generated = (await repository.listLogicalRelationships(database.id))[0]!;
await app.inject({
method: "PATCH",
url: `/catalog/databases/${database.id}/relationships/${generated.id}`,
payload: { status: "excluded" },
});
const second = await app.inject({
method: "POST", url: `/catalog/databases/${database.id}/relationships/rebuild-generated`,
});
expect(second.json()).toEqual({ added: 0, alreadyPresent: 0, excluded: 1, ambiguous: 0 });
});
test("validates relationship requests and keeps physical relationships read-only", async () => {
const { app, repository, database } = await setup();
await seedCatalog(repository, database);
const physical = (await repository.listRelationships(database.id))[0]!;
const invalid = await app.inject({
method: "POST",
url: `/catalog/databases/${database.id}/relationships`,
payload: { sourceColumnId: physical.columns[0].sourceColumnId, targetColumnId: physical.columns[0].targetColumnId, generated: true },
});
expect(invalid.statusCode).toBe(400);
expect(invalid.json()).toMatchObject({ code: "relationship_request_invalid" });
const readOnly = await app.inject({
method: "PATCH",
url: `/catalog/databases/${database.id}/relationships/${physical.id}`,
payload: { status: "excluded" },
});
expect(readOnly.statusCode).toBe(409);
expect(readOnly.json()).toMatchObject({ code: "relationship_read_only" });
});
test("requires database.manage for relationship map mutations", async () => {
const { app, repository, database } = await setup({ AUTH_MODE: "upstream" });
const observed = snapshot();
observed.relationships = [];
await seedCatalog(repository, database, observed);
const response = await app.inject({
method: "POST",
url: `/catalog/databases/${database.id}/relationships/rebuild-generated`,
headers: {
"x-thoth-principal-issuer": "portal",
"x-thoth-principal-subject": "catalog-reader",
"x-thoth-is-admin": "0",
},
});
expect(response.statusCode).toBe(403);
expect(response.json()).toEqual({ code: "auth_forbidden", error: "This operation is not permitted" });
});
@@ -54,6 +54,7 @@ async function setup() {
list: vi.fn(async () => [revision]),
listCatalog: vi.fn(async () => [{ id: "psd-clinical", name: "Policlinico San Donato", configurationState: "ready", revision }]),
read: vi.fn(async () => ({ workspace, revision })),
readPinned: vi.fn(async () => ({ workspace, workspaceConfigPath: revision.snapshotPath })),
} as unknown as WorkspaceRegistry;
const secretStore = new WorkspaceSecretStore({ root: secretRoot, runtimeRoot, installationId: "test" });
const app = buildApp(loadConfig({ THT_HARNESS_DIR: "/missing", NODE_ENV: "test" }), {
@@ -0,0 +1,196 @@
import { describe, expect, test, vi } from "vitest";
import { EffectiveRelationshipSnapshotProvider } from "../src/catalog/effective-relationship-snapshot.js";
import type { CatalogRelationship, WorkspaceDatabase } from "../src/catalog/types.js";
const timestamp = "2026-08-31T10:00:00.000Z";
function database(): WorkspaceDatabase {
return {
id: "database-1",
workspaceId: "psd",
engine: "postgres",
databaseName: "warehouse",
schema: "public",
version: 1,
createdAt: timestamp,
updatedAt: timestamp,
binding: { transport: "postgres_direct" },
connectionStatus: "reachable",
schemaSyncedVersion: 1,
schemaSyncedAt: timestamp,
};
}
const operations = {
async run<T>(_databaseId: string, operation: () => Promise<T>): Promise<T> {
return await operation();
},
};
function relationship(options: {
id: string;
origin: "physical" | "generated" | "manual";
status?: "active" | "excluded";
sourceTable?: string;
sourceColumns?: string[];
targetTable?: string;
targetColumns?: string[];
}): CatalogRelationship {
const sourceColumns = options.sourceColumns ?? ["user_id"];
const targetColumns = options.targetColumns ?? ["id"];
const columns = sourceColumns.map((sourceColumnName, position) => ({
position: position + 1,
sourceColumnId: `source-${position}`,
sourceColumnName,
targetColumnId: `target-${position}`,
targetColumnName: targetColumns[position],
}));
const common = {
id: options.id,
databaseId: "database-1",
sourceTableId: "source-table",
sourceTableName: options.sourceTable ?? "orders",
targetTableId: "target-table",
targetTableName: options.targetTable ?? "users",
columns,
createdAt: timestamp,
updatedAt: timestamp,
};
if (options.origin === "physical") {
return {
...common,
constraintName: `fk_${options.id}`,
updateRule: "NO ACTION",
deleteRule: "NO ACTION",
deferrable: false,
initiallyDeferred: false,
lastSyncedDatabaseVersion: 1,
lastSyncedAt: timestamp,
origin: "physical",
status: "active",
};
}
return {
...common,
constraintName: null,
updateRule: null,
deleteRule: null,
deferrable: false,
initiallyDeferred: false,
lastSyncedDatabaseVersion: null,
lastSyncedAt: null,
origin: options.origin,
status: options.status ?? "active",
columns: [columns[0]!],
};
}
describe("EffectiveRelationshipSnapshotProvider", () => {
test("returns no projection for a workspace without a Catalog database", async () => {
const list = vi.fn();
const provider = new EffectiveRelationshipSnapshotProvider(
{
get: vi.fn().mockResolvedValue(undefined),
getByWorkspace: vi.fn().mockResolvedValue(undefined),
},
{ list },
operations,
);
await expect(provider.render("legacy")).resolves.toBeUndefined();
expect(list).not.toHaveBeenCalled();
});
test("renders a deterministic active map with physical precedence", async () => {
const duplicateGenerated = relationship({ id: "generated", origin: "generated" });
const physical = relationship({ id: "physical", origin: "physical" });
const compositePhysical = relationship({
id: "physical-composite",
origin: "physical",
sourceTable: "order_lines",
sourceColumns: ["order_id", "tenant_id"],
targetTable: "orders",
targetColumns: ["id", "tenant_id"],
});
const manual = relationship({
id: "manual",
origin: "manual",
sourceTable: "invoices",
sourceColumns: ["customer_id"],
targetTable: "customers",
targetColumns: ["id"],
});
const excluded = relationship({
id: "excluded",
origin: "generated",
status: "excluded",
sourceTable: "invoices",
});
const list = vi.fn().mockResolvedValue([
manual,
duplicateGenerated,
excluded,
physical,
compositePhysical,
]);
const provider = new EffectiveRelationshipSnapshotProvider(
{
get: vi.fn().mockResolvedValue(database()),
getByWorkspace: vi.fn().mockResolvedValue(database()),
},
{ list },
operations,
);
const rendered = await provider.render("psd");
expect(rendered?.endsWith("\n")).toBe(true);
expect(JSON.parse(rendered!)).toEqual({
schemaVersion: 1,
workspaceId: "psd",
relationships: [
{
sourceTable: "invoices",
sourceColumns: ["customer_id"],
targetTable: "customers",
targetColumns: ["id"],
origin: "manual",
},
{
sourceTable: "order_lines",
sourceColumns: ["order_id", "tenant_id"],
targetTable: "orders",
targetColumns: ["id", "tenant_id"],
origin: "physical",
},
{
sourceTable: "orders",
sourceColumns: ["user_id"],
targetTable: "users",
targetColumns: ["id"],
origin: "physical",
},
],
});
expect(list).toHaveBeenCalledWith("database-1");
});
test("fails closed when the Catalog schema is absent or stale", async () => {
const stale = database();
delete stale.schemaSyncedVersion;
delete stale.schemaSyncedAt;
const list = vi.fn();
const provider = new EffectiveRelationshipSnapshotProvider(
{
get: vi.fn().mockResolvedValue(stale),
getByWorkspace: vi.fn().mockResolvedValue(stale),
},
{ list },
operations,
);
await expect(provider.render("psd")).rejects.toThrow(
"effective relationship snapshot requires a current full schema synchronization",
);
expect(list).not.toHaveBeenCalled();
});
});
+57
View File
@@ -517,6 +517,63 @@ test("creates a session from the active immutable workspace revision", async ()
}));
});
test("hands one effective relationship snapshot to both retrieval and Pi", async () => {
const effective = JSON.stringify({
schemaVersion: 1,
workspaceId: "default",
relationships: [],
});
const render = vi.fn(async () => effective);
const acquireWorkspaceRuntime = vi.fn((_workspace: string, relationships?: string) => ({
path: "/runtime/with-relationships.yaml",
workspaceId: "default",
workspaceRevision: "e".repeat(40),
release: vi.fn(),
}));
const searchPack = vi.fn(async () => {});
const createFor = vi.fn(() => ({ bridge: { onClientEvent: () => {} } }));
const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), {
thtRunner: {
sessionNew: async () => ({ id: "effective-map" }),
acquireWorkspaceRuntime,
searchPack,
} as any,
effectiveRelationshipSnapshotProvider: { render } as any,
readiness: { ensure: async () => ({ ok: true }) } as any,
mgr: {
get: () => undefined,
createFor,
configure: async () => {},
start: () => {},
} as any,
getSettings: () => ({ workspace: "default", thinking: "low" }) as any,
});
const response = await app.inject({
method: "POST",
url: "/sessions",
payload: { question: "Which users placed orders?", workspaceId: "default" },
});
expect(response.statusCode).toBe(200);
expect(render).toHaveBeenCalledWith("default");
expect(acquireWorkspaceRuntime).toHaveBeenCalledWith(
expect.stringContaining("/default.yaml"),
effective,
);
expect(createFor).toHaveBeenCalledWith(
"effective-map",
expect.objectContaining({
runtimeConfig: expect.objectContaining({ path: "/runtime/with-relationships.yaml" }),
}),
);
expect(searchPack).toHaveBeenCalledWith(
"Which users placed orders?",
"effective-map",
"/runtime/with-relationships.yaml",
);
});
test("rejects an SSH-only workspace before persisting or starting a session", async () => {
const sessionNew = vi.fn(async () => ({ id: "must-not-exist" }));
const ensure = vi.fn(async () => ({ ok: true }));
@@ -197,6 +197,25 @@ test("ThtRunner uses a vault secret only for the lifetime of its runtime lease",
expect(existsSync(rendered.database.password_file)).toBe(false);
});
test("ThtRunner binds and cleans the effective relationship snapshot with its runtime lease", async () => {
const f = await fixture();
const runner = runnerFor(f);
const relationships = JSON.stringify({
schemaVersion: 1,
workspaceId: "psd-clinical",
relationships: [],
});
const lease = runner.acquireWorkspaceRuntime(f.revision.snapshotPath, relationships);
const rendered = parse(readFileSync(lease.path, "utf8")) as {
paths: { effective_relationships: string };
};
expect(readFileSync(rendered.paths.effective_relationships, "utf8")).toBe(relationships);
lease.release();
expect(existsSync(rendered.paths.effective_relationships)).toBe(false);
});
test("separate runtime leases hand off byte-identical revision Evidence configs accepted by tht", async () => {
const f = await fixture();
const runner = runnerFor(f);