feat: consolidate database management work

Add catalog-owned logical relationships and runtime snapshots, extend the database-management UI and validation coverage, and document the updated operational workflow.

Keep active sensitive-generation status in a tooltip and indicator, and update the layout E2E to follow the history action in its new database-scoped location.

backend/src/catalog/effective-relationship-snapshot.ts

@@ -0,0 +1,93 @@
+import type { CatalogRelationship, CatalogRepository } from "./types.js";
+
+export interface EffectiveRelationshipSnapshotReader {
+  list(databaseId: string): Promise<CatalogRelationship[]>;
+}
+
+export interface EffectiveRelationshipSnapshotCoordinator {
+  run<T>(databaseId: string, operation: () => Promise<T>): Promise<T>;
+}
+
+export class EffectiveRelationshipSnapshotStaleError extends Error {}
+
+interface EffectiveRelationship {
+  sourceTable: string;
+  sourceColumns: string[];
+  targetTable: string;
+  targetColumns: string[];
+  origin: CatalogRelationship["origin"];
+}
+
+interface EffectiveRelationshipSnapshot {
+  schemaVersion: 1;
+  workspaceId: string;
+  relationships: EffectiveRelationship[];
+}
+
+const originRank: Record<CatalogRelationship["origin"], number> = {
+  physical: 0,
+  manual: 1,
+  generated: 2,
+};
+
+function endpointKey(relationship: EffectiveRelationship): string {
+  return [
+    relationship.sourceTable,
+    relationship.sourceColumns.join("\u0000"),
+    relationship.targetTable,
+    relationship.targetColumns.join("\u0000"),
+  ].join("\u0001");
+}
+
+function compareRelationships(left: EffectiveRelationship, right: EffectiveRelationship): number {
+  return endpointKey(left).localeCompare(endpointKey(right))
+    || originRank[left.origin] - originRank[right.origin];
+}
+
+/**
+ * Adapter from the mutable Catalog model to the immutable relationship contract consumed by the
+ * harness. The returned JSON is a deterministic projection, never an authored second store.
+ */
+export class EffectiveRelationshipSnapshotProvider {
+  constructor(
+    private readonly repository: Pick<CatalogRepository, "get" | "getByWorkspace">,
+    private readonly relationships: EffectiveRelationshipSnapshotReader,
+    private readonly operations: EffectiveRelationshipSnapshotCoordinator,
+  ) {}
+
+  async render(workspaceId: string): Promise<string | undefined> {
+    const database = await this.repository.getByWorkspace(workspaceId);
+    if (!database) return undefined;
+    return await this.operations.run(database.id, async () => {
+      const current = await this.repository.get(database.id);
+      if (!current || current.schemaSyncedVersion !== current.version) {
+        throw new EffectiveRelationshipSnapshotStaleError(
+          "effective relationship snapshot requires a current full schema synchronization",
+        );
+      }
+      const projected = (await this.relationships.list(database.id))
+        .filter((relationship) => relationship.status === "active")
+        .map((relationship): EffectiveRelationship => ({
+          sourceTable: relationship.sourceTableName,
+          sourceColumns: relationship.columns.map((column) => column.sourceColumnName),
+          targetTable: relationship.targetTableName,
+          targetColumns: relationship.columns.map((column) => column.targetColumnName),
+          origin: relationship.origin,
+        }))
+        .sort(compareRelationships);
+
+      const seen = new Set<string>();
+      const snapshot: EffectiveRelationshipSnapshot = {
+        schemaVersion: 1,
+        workspaceId,
+        relationships: projected.filter((relationship) => {
+          const key = endpointKey(relationship);
+          if (seen.has(key)) return false;
+          seen.add(key);
+          return true;
+        }),
+      };
+      return `${JSON.stringify(snapshot, null, 2)}\n`;
+    });
+  }
+}

backend/src/catalog/logical-relationship-service.ts

@@ -0,0 +1,260 @@
+import type {
+  CatalogLogicalRelationship,
+  CatalogLogicalRelationshipCandidate,
+  CatalogLogicalRelationshipContext,
+  CatalogLogicalRelationshipEndpoint,
+  CatalogRelationship,
+  CatalogRepository,
+} from "./types.js";
+
+export class LogicalRelationshipDatabaseNotFoundError extends Error {}
+export class LogicalRelationshipDuplicateError extends Error {}
+export class LogicalRelationshipNotFoundError extends Error {}
+export class LogicalRelationshipReadOnlyError extends Error {}
+export class LogicalRelationshipSchemaStaleError extends Error {}
+export class LogicalRelationshipTargetNotUniqueError extends Error {}
+export class LogicalRelationshipTypeIncompatibleError extends Error {}
+export class LogicalRelationshipColumnNotFoundError extends Error {
+  constructor(readonly field: "sourceColumnId" | "targetColumnId") {
+    super(`Catalog column '${field}' was not found`);
+  }
+}
+
+export interface RebuildGeneratedRelationshipsResult {
+  added: number;
+  alreadyPresent: number;
+  excluded: number;
+  ambiguous: number;
+}
+
+function identifierTokens(value: string): string[] {
+  return value
+    .replace(/([a-z0-9])([A-Z])/g, "$1_$2")
+    .toLowerCase()
+    .split(/[^a-z0-9]+/)
+    .filter(Boolean);
+}
+
+function singularWord(value: string): string {
+  if (value.length > 4 && value.endsWith("ies")) return `${value.slice(0, -3)}y`;
+  if (value.length > 4 && /(ches|shes|xes|zes|ses)$/.test(value)) return value.slice(0, -2);
+  if (value.length > 3 && value.endsWith("s") && !/(ss|us)$/.test(value)) return value.slice(0, -1);
+  return value;
+}
+
+function tableAliases(tableName: string): string[] {
+  const tokens = identifierTokens(tableName);
+  if (tokens.length === 0) return [];
+  const normalized = tokens.join("_");
+  const singular = [...tokens];
+  singular[singular.length - 1] = singularWord(singular[singular.length - 1]);
+  return [...new Set([normalized, singular.join("_")])];
+}
+
+const GENERIC_PRIMARY_KEY_NAMES = new Set(["id", "key", "code", "pk"]);
+
+function nameMatches(
+  source: CatalogLogicalRelationshipEndpoint,
+  target: CatalogLogicalRelationshipEndpoint,
+): boolean {
+  const sourceName = identifierTokens(source.columnName).join("_");
+  const targetName = identifierTokens(target.columnName).join("_");
+  if (!sourceName || !targetName) return false;
+  const expected = new Set<string>();
+  if (!GENERIC_PRIMARY_KEY_NAMES.has(targetName)) expected.add(targetName);
+  for (const alias of tableAliases(target.tableName)) {
+    expected.add(`${alias}_${targetName}`);
+    expected.add(`${alias.replaceAll("_", "")}${targetName.replaceAll("_", "")}`);
+    if (targetName === "id" || targetName === "pk") expected.add(alias);
+  }
+  return expected.has(sourceName);
+}
+
+function canonicalDataType(value: string): string {
+  const normalized = value.trim().toLowerCase().replace(/\s+/g, " ");
+  const arraySuffix = normalized.endsWith("[]") ? "[]" : "";
+  const base = arraySuffix ? normalized.slice(0, -2) : normalized;
+  const withoutModifier = base.replace(/\([^)]*\)/g, "").trim();
+  const aliases: Record<string, string> = {
+    int2: "smallint",
+    smallserial: "smallint",
+    int4: "integer",
+    int: "integer",
+    serial: "integer",
+    int8: "bigint",
+    bigserial: "bigint",
+    decimal: "numeric",
+    varchar: "text",
+    "character varying": "text",
+    bool: "boolean",
+    "timestamp without time zone": "timestamp",
+    "timestamp with time zone": "timestamptz",
+    "time without time zone": "time",
+    "time with time zone": "timetz",
+  };
+  return `${aliases[withoutModifier] ?? withoutModifier}${arraySuffix}`;
+}
+
+function typesCompatible(left: string, right: string): boolean {
+  return canonicalDataType(left) === canonicalDataType(right);
+}
+
+function pairKey(sourceColumnId: string, targetColumnId: string): string {
+  return `${sourceColumnId}\u0000${targetColumnId}`;
+}
+
+function relationshipPair(relationship: CatalogLogicalRelationship): CatalogLogicalRelationshipCandidate {
+  return {
+    sourceColumnId: relationship.columns[0].sourceColumnId,
+    targetColumnId: relationship.columns[0].targetColumnId,
+  };
+}
+
+function relationshipSortKey(relationship: CatalogRelationship): string {
+  const sourceColumns = relationship.columns.map((column) => column.sourceColumnName).join(",");
+  const targetColumns = relationship.columns.map((column) => column.targetColumnName).join(",");
+  return [
+    relationship.sourceTableName,
+    sourceColumns,
+    relationship.targetTableName,
+    targetColumns,
+    relationship.origin,
+  ].join("\u0000");
+}
+
+export class CatalogLogicalRelationshipService {
+  constructor(private readonly repository: CatalogRepository) {}
+
+  async list(databaseId: string): Promise<CatalogRelationship[]> {
+    if (!(await this.repository.get(databaseId))) throw new LogicalRelationshipDatabaseNotFoundError();
+    const relationships: CatalogRelationship[] = [
+      ...await this.repository.listRelationships(databaseId),
+      ...await this.repository.listLogicalRelationships(databaseId),
+    ];
+    return relationships.sort((left, right) => relationshipSortKey(left).localeCompare(relationshipSortKey(right)));
+  }
+
+  async addManual(
+    databaseId: string,
+    sourceColumnId: string,
+    targetColumnId: string,
+  ): Promise<CatalogLogicalRelationship> {
+    const context = await this.requiredContext(databaseId);
+    const source = context.endpoints.find((endpoint) => endpoint.columnId === sourceColumnId);
+    if (!source) throw new LogicalRelationshipColumnNotFoundError("sourceColumnId");
+    const target = context.endpoints.find((endpoint) => endpoint.columnId === targetColumnId);
+    if (!target) throw new LogicalRelationshipColumnNotFoundError("targetColumnId");
+    if (sourceColumnId === targetColumnId
+      || target.primaryKeyPosition === null
+      || target.tablePrimaryKeyColumnCount !== 1) {
+      throw new LogicalRelationshipTargetNotUniqueError();
+    }
+    if (!typesCompatible(source.dataType, target.dataType)) {
+      throw new LogicalRelationshipTypeIncompatibleError();
+    }
+    const key = pairKey(sourceColumnId, targetColumnId);
+    if (context.physicalPairs.some((pair) => pairKey(pair.sourceColumnId, pair.targetColumnId) === key)
+      || context.logicalRelationships.some((relationship) => {
+        const pair = relationshipPair(relationship);
+        return pairKey(pair.sourceColumnId, pair.targetColumnId) === key;
+      })) throw new LogicalRelationshipDuplicateError();
+    const created = await this.repository.insertLogicalRelationship(
+      databaseId,
+      sourceColumnId,
+      targetColumnId,
+      false,
+    );
+    if (!created) throw new LogicalRelationshipDuplicateError();
+    return created;
+  }
+
+  async rebuildGenerated(databaseId: string): Promise<RebuildGeneratedRelationshipsResult> {
+    const context = await this.requiredContext(databaseId);
+    const targets = context.endpoints.filter((endpoint) => (
+      endpoint.primaryKeyPosition !== null && endpoint.tablePrimaryKeyColumnCount === 1
+    ));
+    const physical = new Set(context.physicalPairs.map((pair) => pairKey(pair.sourceColumnId, pair.targetColumnId)));
+    const active = new Set<string>();
+    const excluded = new Set<string>();
+    for (const relationship of context.logicalRelationships) {
+      const pair = relationshipPair(relationship);
+      (relationship.status === "excluded" ? excluded : active)
+        .add(pairKey(pair.sourceColumnId, pair.targetColumnId));
+    }
+
+    const pending: CatalogLogicalRelationshipCandidate[] = [];
+    let alreadyPresent = 0;
+    let excludedCount = 0;
+    let ambiguous = 0;
+    const dimTimeTargets = targets.filter((target) => (
+      identifierTokens(target.tableName).join("_") === "dim_time"
+    ));
+    const sources = context.endpoints.filter((endpoint) => (
+      endpoint.primaryKeyPosition === null || endpoint.tablePrimaryKeyColumnCount > 1
+    ));
+    for (const source of sources) {
+      const sourceName = identifierTokens(source.columnName).join("_");
+      const isTimeKey = sourceName.endsWith("time_key")
+        && identifierTokens(source.tableName).join("_") !== "dim_time";
+      const candidates = isTimeKey ? dimTimeTargets : targets;
+      const matches = candidates.filter((target) => (
+        target.columnId !== source.columnId
+        && typesCompatible(source.dataType, target.dataType)
+        && (isTimeKey || nameMatches(source, target))
+      ));
+      if (matches.length > 1) {
+        ambiguous += 1;
+        continue;
+      }
+      if (matches.length === 0) continue;
+      const candidate = { sourceColumnId: source.columnId, targetColumnId: matches[0].columnId };
+      const key = pairKey(candidate.sourceColumnId, candidate.targetColumnId);
+      if (physical.has(key) || active.has(key)) {
+        alreadyPresent += 1;
+      } else if (excluded.has(key)) {
+        excludedCount += 1;
+      } else {
+        pending.push(candidate);
+      }
+    }
+    const added = await this.repository.insertGeneratedLogicalRelationships(databaseId, pending);
+    alreadyPresent += pending.length - added;
+    return { added, alreadyPresent, excluded: excludedCount, ambiguous };
+  }
+
+  async setStatus(
+    databaseId: string,
+    relationshipId: string,
+    status: CatalogLogicalRelationship["status"],
+  ): Promise<CatalogLogicalRelationship> {
+    if (!(await this.repository.get(databaseId))) throw new LogicalRelationshipDatabaseNotFoundError();
+    const updated = await this.repository.setLogicalRelationshipStatus(databaseId, relationshipId, status);
+    if (updated) return updated;
+    await this.assertNotPhysical(databaseId, relationshipId);
+    throw new LogicalRelationshipNotFoundError();
+  }
+
+  async deletePermanently(databaseId: string, relationshipId: string): Promise<void> {
+    if (!(await this.repository.get(databaseId))) throw new LogicalRelationshipDatabaseNotFoundError();
+    if (await this.repository.deleteLogicalRelationship(databaseId, relationshipId)) return;
+    await this.assertNotPhysical(databaseId, relationshipId);
+    throw new LogicalRelationshipNotFoundError();
+  }
+
+  private async requiredContext(databaseId: string): Promise<CatalogLogicalRelationshipContext> {
+    const database = await this.repository.get(databaseId);
+    if (!database) throw new LogicalRelationshipDatabaseNotFoundError();
+    if (database.schemaSyncedVersion !== database.version) {
+      throw new LogicalRelationshipSchemaStaleError();
+    }
+    const context = await this.repository.getLogicalRelationshipContext(databaseId);
+    if (!context) throw new LogicalRelationshipDatabaseNotFoundError();
+    return context;
+  }
+
+  private async assertNotPhysical(databaseId: string, relationshipId: string): Promise<void> {
+    if ((await this.repository.listRelationships(databaseId)).some((relationship) => relationship.id === relationshipId)) {
+      throw new LogicalRelationshipReadOnlyError();
+    }
+  }
+}

backend/src/catalog/memory-repository.ts

@@ -14,7 +14,10 @@ import {
   type CatalogDatabaseMetadataDeleteTarget,
   type CatalogMetadataDeleteCounts,
   type CatalogMetrics,
-  type CatalogRelationship,
+  type CatalogLogicalRelationship,
+  type CatalogLogicalRelationshipCandidate,
+  type CatalogLogicalRelationshipContext,
+  type CatalogPhysicalRelationship,
   type CatalogSchemaDiff,
   type CatalogSyncCounts,
   type CatalogSyncEvent,
@@ -49,7 +52,8 @@ export class MemoryCatalogRepository implements CatalogRepository {
   private readonly records = new Map<string, WorkspaceDatabase>();
   private readonly tables = new Map<string, CatalogTable>();
   private readonly columns = new Map<string, CatalogColumn>();
-  private readonly relationships = new Map<string, CatalogRelationship>();
+  private readonly relationships = new Map<string, CatalogPhysicalRelationship>();
+  private readonly logicalRelationships = new Map<string, CatalogLogicalRelationship>();
   private readonly descriptionGenerationRuns = new Map<string, DescriptionGenerationRun>();
   private readonly descriptionGenerationEvents = new Map<string, DescriptionGenerationEvent[]>();
   private readonly sensitiveDataSuggestionRuns = new Map<string, SensitiveDataSuggestionRun>();
@@ -171,6 +175,9 @@ export class MemoryCatalogRepository implements CatalogRepository {
     for (const [relationshipId, relationship] of this.relationships) {
       if (relationship.databaseId === id) this.relationships.delete(relationshipId);
     }
+    for (const [relationshipId, relationship] of this.logicalRelationships) {
+      if (relationship.databaseId === id) this.logicalRelationships.delete(relationshipId);
+    }
     for (const [runId, run] of this.descriptionGenerationRuns) {
       if (run.databaseId !== id) continue;
       this.descriptionGenerationRuns.delete(runId);
@@ -527,12 +534,138 @@ export class MemoryCatalogRepository implements CatalogRepository {
       .map((event) => structuredClone(event));
   }
 
-  async listRelationships(databaseId: string): Promise<CatalogRelationship[]> {
+  async listRelationships(databaseId: string): Promise<CatalogPhysicalRelationship[]> {
     return [...this.relationships.values()].filter((relationship) => relationship.databaseId === databaseId)
       .sort((a, b) => `${a.sourceTableName}.${a.constraintName}`.localeCompare(`${b.sourceTableName}.${b.constraintName}`))
       .map((relationship) => structuredClone(relationship));
   }
 
+  async listLogicalRelationships(databaseId: string): Promise<CatalogLogicalRelationship[]> {
+    return [...this.logicalRelationships.values()]
+      .filter((relationship) => relationship.databaseId === databaseId)
+      .sort((a, b) => {
+        const left = `${a.sourceTableName}.${a.columns[0].sourceColumnName}.${a.targetTableName}.${a.columns[0].targetColumnName}`;
+        const right = `${b.sourceTableName}.${b.columns[0].sourceColumnName}.${b.targetTableName}.${b.columns[0].targetColumnName}`;
+        return left.localeCompare(right);
+      })
+      .map((relationship) => structuredClone(relationship));
+  }
+
+  async getLogicalRelationshipContext(
+    databaseId: string,
+  ): Promise<CatalogLogicalRelationshipContext | undefined> {
+    if (!this.records.has(databaseId)) return undefined;
+    const tables = [...this.tables.values()].filter((table) => table.databaseId === databaseId);
+    const tableById = new Map(tables.map((table) => [table.id, table]));
+    const columns = [...this.columns.values()].filter((column) => tableById.has(column.tableId));
+    const primaryKeyCounts = new Map<string, number>();
+    for (const column of columns) {
+      if (column.primaryKeyPosition !== null) {
+        primaryKeyCounts.set(column.tableId, (primaryKeyCounts.get(column.tableId) ?? 0) + 1);
+      }
+    }
+    return {
+      endpoints: columns.map((column) => ({
+        columnId: column.id,
+        columnName: column.name,
+        tableId: column.tableId,
+        tableName: tableById.get(column.tableId)!.name,
+        dataType: column.dataType,
+        primaryKeyPosition: column.primaryKeyPosition,
+        tablePrimaryKeyColumnCount: primaryKeyCounts.get(column.tableId) ?? 0,
+      })),
+      physicalPairs: [...this.relationships.values()]
+        .filter((relationship) => relationship.databaseId === databaseId)
+        .flatMap((relationship) => relationship.columns.map((column) => ({
+          sourceColumnId: column.sourceColumnId,
+          targetColumnId: column.targetColumnId,
+        }))),
+      logicalRelationships: await this.listLogicalRelationships(databaseId),
+    };
+  }
+
+  async insertLogicalRelationship(
+    databaseId: string,
+    sourceColumnId: string,
+    targetColumnId: string,
+    generated: boolean,
+  ): Promise<CatalogLogicalRelationship | undefined> {
+    if ([...this.logicalRelationships.values()].some((relationship) => (
+      relationship.databaseId === databaseId
+      && relationship.columns[0].sourceColumnId === sourceColumnId
+      && relationship.columns[0].targetColumnId === targetColumnId
+    ))) return undefined;
+    const sourceColumn = this.columns.get(sourceColumnId);
+    const targetColumn = this.columns.get(targetColumnId);
+    const sourceTable = sourceColumn ? this.tables.get(sourceColumn.tableId) : undefined;
+    const targetTable = targetColumn ? this.tables.get(targetColumn.tableId) : undefined;
+    if (!sourceColumn || !targetColumn || !sourceTable || !targetTable
+      || sourceTable.databaseId !== databaseId || targetTable.databaseId !== databaseId
+      || sourceColumnId === targetColumnId) return undefined;
+    const now = new Date().toISOString();
+    const relationship: CatalogLogicalRelationship = {
+      id: randomUUID(),
+      databaseId,
+      constraintName: null,
+      sourceTableId: sourceTable.id,
+      sourceTableName: sourceTable.name,
+      targetTableId: targetTable.id,
+      targetTableName: targetTable.name,
+      updateRule: null,
+      deleteRule: null,
+      deferrable: false,
+      initiallyDeferred: false,
+      columns: [{
+        position: 1,
+        sourceColumnId,
+        sourceColumnName: sourceColumn.name,
+        targetColumnId,
+        targetColumnName: targetColumn.name,
+      }],
+      lastSyncedDatabaseVersion: null,
+      lastSyncedAt: null,
+      createdAt: now,
+      updatedAt: now,
+      origin: generated ? "generated" : "manual",
+      status: "active",
+    };
+    this.logicalRelationships.set(relationship.id, relationship);
+    return structuredClone(relationship);
+  }
+
+  async insertGeneratedLogicalRelationships(
+    databaseId: string,
+    candidates: readonly CatalogLogicalRelationshipCandidate[],
+  ): Promise<number> {
+    let added = 0;
+    for (const candidate of candidates) {
+      if (await this.insertLogicalRelationship(
+        databaseId,
+        candidate.sourceColumnId,
+        candidate.targetColumnId,
+        true,
+      )) added += 1;
+    }
+    return added;
+  }
+
+  async setLogicalRelationshipStatus(
+    databaseId: string,
+    relationshipId: string,
+    status: CatalogLogicalRelationship["status"],
+  ): Promise<CatalogLogicalRelationship | undefined> {
+    const current = this.logicalRelationships.get(relationshipId);
+    if (!current || current.databaseId !== databaseId) return undefined;
+    const updated = { ...current, status, updatedAt: new Date().toISOString() };
+    this.logicalRelationships.set(relationshipId, updated);
+    return structuredClone(updated);
+  }
+
+  async deleteLogicalRelationship(databaseId: string, relationshipId: string): Promise<boolean> {
+    const current = this.logicalRelationships.get(relationshipId);
+    return Boolean(current?.databaseId === databaseId && this.logicalRelationships.delete(relationshipId));
+  }
+
   async deleteDatabaseMetadata(
     databaseIds: readonly string[],
     target: CatalogDatabaseMetadataDeleteTarget,
@@ -574,6 +707,12 @@ export class MemoryCatalogRepository implements CatalogRepository {
       const columns = [...this.columns.values()].filter((column) => selected.has(column.tableId));
       const deletedColumnIds = new Set(columns.map((column) => column.id));
       for (const column of columns) this.columns.delete(column.id);
+      for (const relationship of [...this.logicalRelationships.values()]) {
+        const pair = relationship.columns[0];
+        if (deletedColumnIds.has(pair.sourceColumnId) || deletedColumnIds.has(pair.targetColumnId)) {
+          this.logicalRelationships.delete(relationship.id);
+        }
+      }
       for (const [relationshipId, relationship] of this.relationships) {
         if (relationship.databaseId !== databaseId) continue;
         this.relationships.set(relationshipId, {
@@ -857,6 +996,8 @@ export class MemoryCatalogRepository implements CatalogRepository {
             lastSyncedAt: now,
             createdAt: current?.createdAt ?? now,
             updatedAt: comparable === nextComparable ? (current?.updatedAt ?? now) : now,
+            origin: "physical",
+            status: "active",
           });
           if (!current) created += 1;
           else if (comparable !== nextComparable) updated += 1;
@@ -1073,6 +1214,12 @@ export class MemoryCatalogRepository implements CatalogRepository {
         this.relationships.delete(relationship.id);
       }
     }
+    for (const relationship of [...this.logicalRelationships.values()]) {
+      const pair = relationship.columns[0];
+      if (pair.sourceColumnId === columnId || pair.targetColumnId === columnId) {
+        this.logicalRelationships.delete(relationship.id);
+      }
+    }
   }
 
   private markCatalogIncomplete(databaseIds: readonly string[]): void {

backend/src/catalog/migrate.ts

@@ -10,6 +10,7 @@ import * as catalogRuntimeSequencePrivilegesMigration from "./migrations/004_cat
 import * as descriptionGenerationRunsMigration from "./migrations/005_description_generation_runs.js";
 import * as sensitiveDataFlagMigration from "./migrations/006_sensitive_data_flag.js";
 import * as sensitiveDataSuggestionRunsMigration from "./migrations/007_sensitive_data_suggestion_runs.js";
+import * as catalogLogicalRelationshipsMigration from "./migrations/008_catalog_logical_relationships.js";
 
 const connectionString = process.env.THT_CATALOG_MIGRATOR_DATABASE_URL;
 const host = process.env.THT_CATALOG_DB_HOST;
@@ -42,6 +43,7 @@ const provider: MigrationProvider = {
       "005_description_generation_runs": descriptionGenerationRunsMigration,
       "006_sensitive_data_flag": sensitiveDataFlagMigration,
       "007_sensitive_data_suggestion_runs": sensitiveDataSuggestionRunsMigration,
+      "008_catalog_logical_relationships": catalogLogicalRelationshipsMigration,
     };
   },
 };

backend/src/catalog/migrations/008_catalog_logical_relationships.ts

@@ -0,0 +1,35 @@
+import { type Kysely, sql } from "kysely";
+import type { CatalogDatabase } from "../repository.js";
+
+export async function up(db: Kysely<CatalogDatabase>): Promise<void> {
+  await db.schema.createTable("catalog_logical_relationships")
+    .addColumn("id", "uuid", (column) => column.primaryKey())
+    .addColumn("database_id", "uuid", (column) => column.notNull()
+      .references("workspace_databases.id").onDelete("cascade"))
+    .addColumn("source_column_id", "uuid", (column) => column.notNull()
+      .references("catalog_columns.id").onDelete("cascade"))
+    .addColumn("target_column_id", "uuid", (column) => column.notNull()
+      .references("catalog_columns.id").onDelete("cascade"))
+    .addColumn("generated", "boolean", (column) => column.notNull().defaultTo(false))
+    .addColumn("deleted_at", "timestamptz")
+    .addColumn("created_at", "timestamptz", (column) => column.notNull().defaultTo(sql`now()`))
+    .addColumn("updated_at", "timestamptz", (column) => column.notNull().defaultTo(sql`now()`))
+    .addUniqueConstraint(
+      "catalog_logical_relationships_endpoint_key",
+      ["database_id", "source_column_id", "target_column_id"],
+    )
+    .addCheckConstraint(
+      "catalog_logical_relationships_distinct_columns_check",
+      sql`source_column_id <> target_column_id`,
+    )
+    .execute();
+
+  await db.schema.createIndex("catalog_logical_relationships_database_deleted_idx")
+    .on("catalog_logical_relationships")
+    .columns(["database_id", "deleted_at"])
+    .execute();
+}
+
+export async function down(db: Kysely<CatalogDatabase>): Promise<void> {
+  await db.schema.dropTable("catalog_logical_relationships").execute();
+}

backend/src/catalog/repository.ts

@@ -23,7 +23,10 @@ import {
   type CatalogDatabaseMetadataDeleteTarget,
   type CatalogMetadataDeleteCounts,
   type CatalogMetrics,
-  type CatalogRelationship,
+  type CatalogLogicalRelationship,
+  type CatalogLogicalRelationshipCandidate,
+  type CatalogLogicalRelationshipContext,
+  type CatalogPhysicalRelationship,
   type CatalogSchemaDiff,
   type CatalogSyncCounts,
   type CatalogSyncEvent,
@@ -145,6 +148,17 @@ interface CatalogRelationshipColumnTable {
   targetColumnId: string;
 }
 
+interface CatalogLogicalRelationshipTable {
+  id: string;
+  databaseId: string;
+  sourceColumnId: string;
+  targetColumnId: string;
+  generated: Generated<boolean>;
+  deletedAt: Timestamp | null;
+  createdAt: Timestamp;
+  updatedAt: Timestamp;
+}
+
 interface DescriptionGenerationRunTable {
   id: string;
   databaseId: string;
@@ -241,6 +255,7 @@ export interface CatalogDatabase {
   catalogColumns: CatalogColumnTable;
   catalogRelationships: CatalogRelationshipTable;
   catalogRelationshipColumns: CatalogRelationshipColumnTable;
+  catalogLogicalRelationships: CatalogLogicalRelationshipTable;
   descriptionGenerationRuns: DescriptionGenerationRunTable;
   descriptionGenerationEvents: DescriptionGenerationEventTable;
   sensitiveDataSuggestionRuns: SensitiveDataSuggestionRunTable;
@@ -1022,7 +1037,7 @@ export class KyselyCatalogRepository implements CatalogRepository {
     return rows.map(serializeSensitiveDataSuggestionEvent);
   }
 
-  async listRelationships(databaseId: string): Promise<CatalogRelationship[]> {
+  async listRelationships(databaseId: string): Promise<CatalogPhysicalRelationship[]> {
     const rows = await this.db.selectFrom("catalogRelationships as relationship")
       .innerJoin("catalogTables as sourceTable", "sourceTable.id", "relationship.sourceTableId")
       .innerJoin("catalogTables as targetTable", "targetTable.id", "relationship.targetTableId")
@@ -1049,7 +1064,7 @@ export class KyselyCatalogRepository implements CatalogRepository {
       .where("pair.relationshipId", "in", rows.map((row) => row.id))
       .orderBy("pair.relationshipId").orderBy("pair.position")
       .execute();
-    const byRelationship = new Map<string, CatalogRelationship["columns"]>();
+    const byRelationship = new Map<string, CatalogPhysicalRelationship["columns"]>();
     for (const pair of pairs) {
       const items = byRelationship.get(pair.relationshipId) ?? [];
       items.push(pair);
@@ -1061,9 +1076,160 @@ export class KyselyCatalogRepository implements CatalogRepository {
       lastSyncedAt: row.lastSyncedAt === null ? null : new Date(row.lastSyncedAt).toISOString(),
       createdAt: new Date(row.createdAt).toISOString(),
       updatedAt: new Date(row.updatedAt).toISOString(),
+      origin: "physical" as const,
+      status: "active" as const,
     }));
   }
 
+  async listLogicalRelationships(databaseId: string): Promise<CatalogLogicalRelationship[]> {
+    const rows = await this.db.selectFrom("catalogLogicalRelationships as relationship")
+      .innerJoin("catalogColumns as sourceColumn", "sourceColumn.id", "relationship.sourceColumnId")
+      .innerJoin("catalogTables as sourceTable", "sourceTable.id", "sourceColumn.tableId")
+      .innerJoin("catalogColumns as targetColumn", "targetColumn.id", "relationship.targetColumnId")
+      .innerJoin("catalogTables as targetTable", "targetTable.id", "targetColumn.tableId")
+      .select([
+        "relationship.id", "relationship.databaseId", "relationship.generated",
+        "relationship.deletedAt", "relationship.createdAt", "relationship.updatedAt",
+        "sourceTable.id as sourceTableId", "sourceTable.name as sourceTableName",
+        "sourceColumn.id as sourceColumnId", "sourceColumn.name as sourceColumnName",
+        "targetTable.id as targetTableId", "targetTable.name as targetTableName",
+        "targetColumn.id as targetColumnId", "targetColumn.name as targetColumnName",
+      ])
+      .where("relationship.databaseId", "=", databaseId)
+      .orderBy("sourceTable.name")
+      .orderBy("sourceColumn.name")
+      .orderBy("targetTable.name")
+      .orderBy("targetColumn.name")
+      .execute();
+    return rows.map((row) => ({
+      id: row.id,
+      databaseId: row.databaseId,
+      constraintName: null,
+      sourceTableId: row.sourceTableId,
+      sourceTableName: row.sourceTableName,
+      targetTableId: row.targetTableId,
+      targetTableName: row.targetTableName,
+      updateRule: null,
+      deleteRule: null,
+      deferrable: false,
+      initiallyDeferred: false,
+      columns: [{
+        position: 1,
+        sourceColumnId: row.sourceColumnId,
+        sourceColumnName: row.sourceColumnName,
+        targetColumnId: row.targetColumnId,
+        targetColumnName: row.targetColumnName,
+      }],
+      lastSyncedDatabaseVersion: null,
+      lastSyncedAt: null,
+      createdAt: new Date(row.createdAt).toISOString(),
+      updatedAt: new Date(row.updatedAt).toISOString(),
+      origin: row.generated ? "generated" : "manual",
+      status: row.deletedAt === null ? "active" : "excluded",
+    }));
+  }
+
+  async getLogicalRelationshipContext(
+    databaseId: string,
+  ): Promise<CatalogLogicalRelationshipContext | undefined> {
+    if (!(await selectOne(this.db, databaseId))) return undefined;
+    const columns = await this.db.selectFrom("catalogColumns as column")
+      .innerJoin("catalogTables as table", "table.id", "column.tableId")
+      .select([
+        "column.id as columnId", "column.name as columnName", "column.dataType",
+        "column.primaryKeyPosition", "table.id as tableId", "table.name as tableName",
+      ])
+      .where("table.databaseId", "=", databaseId)
+      .orderBy("table.name")
+      .orderBy("column.ordinalPosition")
+      .execute();
+    const primaryKeyCounts = new Map<string, number>();
+    for (const column of columns) {
+      if (column.primaryKeyPosition !== null) {
+        primaryKeyCounts.set(column.tableId, (primaryKeyCounts.get(column.tableId) ?? 0) + 1);
+      }
+    }
+    const physicalPairs = await this.db.selectFrom("catalogRelationshipColumns as pair")
+      .innerJoin("catalogRelationships as relationship", "relationship.id", "pair.relationshipId")
+      .select(["pair.sourceColumnId", "pair.targetColumnId"])
+      .where("relationship.databaseId", "=", databaseId)
+      .execute();
+    return {
+      endpoints: columns.map((column) => ({
+        ...column,
+        tablePrimaryKeyColumnCount: primaryKeyCounts.get(column.tableId) ?? 0,
+      })),
+      physicalPairs,
+      logicalRelationships: await this.listLogicalRelationships(databaseId),
+    };
+  }
+
+  async insertLogicalRelationship(
+    databaseId: string,
+    sourceColumnId: string,
+    targetColumnId: string,
+    generated: boolean,
+  ): Promise<CatalogLogicalRelationship | undefined> {
+    const id = randomUUID();
+    const inserted = await this.db.insertInto("catalogLogicalRelationships").values({
+      id, databaseId, sourceColumnId, targetColumnId, generated,
+    }).onConflict((conflict) => conflict
+      .columns(["databaseId", "sourceColumnId", "targetColumnId"])
+      .doNothing())
+      .returning("id")
+      .executeTakeFirst();
+    if (!inserted) return undefined;
+    return (await this.listLogicalRelationships(databaseId)).find((item) => item.id === id);
+  }
+
+  async insertGeneratedLogicalRelationships(
+    databaseId: string,
+    candidates: readonly CatalogLogicalRelationshipCandidate[],
+  ): Promise<number> {
+    if (candidates.length === 0) return 0;
+    return await this.db.transaction().execute(async (trx) => {
+      let added = 0;
+      for (const candidate of candidates) {
+        const inserted = await trx.insertInto("catalogLogicalRelationships").values({
+          id: randomUUID(),
+          databaseId,
+          sourceColumnId: candidate.sourceColumnId,
+          targetColumnId: candidate.targetColumnId,
+          generated: true,
+        }).onConflict((conflict) => conflict
+          .columns(["databaseId", "sourceColumnId", "targetColumnId"])
+          .doNothing())
+          .returning("id")
+          .executeTakeFirst();
+        if (inserted) added += 1;
+      }
+      return added;
+    });
+  }
+
+  async setLogicalRelationshipStatus(
+    databaseId: string,
+    relationshipId: string,
+    status: CatalogLogicalRelationship["status"],
+  ): Promise<CatalogLogicalRelationship | undefined> {
+    const updated = await this.db.updateTable("catalogLogicalRelationships")
+      .set({ deletedAt: status === "excluded" ? sql`now()` : null, updatedAt: sql`now()` })
+      .where("databaseId", "=", databaseId)
+      .where("id", "=", relationshipId)
+      .returning("id")
+      .executeTakeFirst();
+    if (!updated) return undefined;
+    return (await this.listLogicalRelationships(databaseId)).find((item) => item.id === relationshipId);
+  }
+
+  async deleteLogicalRelationship(databaseId: string, relationshipId: string): Promise<boolean> {
+    const result = await this.db.deleteFrom("catalogLogicalRelationships")
+      .where("databaseId", "=", databaseId)
+      .where("id", "=", relationshipId)
+      .executeTakeFirst();
+    return result.numDeletedRows > 0n;
+  }
+
   async deleteDatabaseMetadata(
     databaseIds: readonly string[],
     target: CatalogDatabaseMetadataDeleteTarget,
@@ -1621,7 +1787,13 @@ export class UnavailableCatalogRepository implements CatalogRepository {
   async updateSensitiveDataSuggestionRun(): Promise<SensitiveDataSuggestionRun | undefined> { return this.fail(); }
   async appendSensitiveDataSuggestionEvent(): Promise<SensitiveDataSuggestionEvent> { return this.fail(); }
   async listSensitiveDataSuggestionEvents(): Promise<SensitiveDataSuggestionEvent[]> { return this.fail(); }
-  async listRelationships(): Promise<CatalogRelationship[]> { return this.fail(); }
+  async listRelationships(): Promise<CatalogPhysicalRelationship[]> { return this.fail(); }
+  async listLogicalRelationships(): Promise<CatalogLogicalRelationship[]> { return this.fail(); }
+  async getLogicalRelationshipContext(): Promise<CatalogLogicalRelationshipContext | undefined> { return this.fail(); }
+  async insertLogicalRelationship(): Promise<CatalogLogicalRelationship | undefined> { return this.fail(); }
+  async insertGeneratedLogicalRelationships(): Promise<number> { return this.fail(); }
+  async setLogicalRelationshipStatus(): Promise<CatalogLogicalRelationship | undefined> { return this.fail(); }
+  async deleteLogicalRelationship(): Promise<boolean> { return this.fail(); }
   async deleteDatabaseMetadata(): Promise<CatalogMetadataDeleteCounts | undefined> { return this.fail(); }
   async deleteTableMetadata(): Promise<CatalogMetadataDeleteCounts | undefined> { return this.fail(); }
   async planSchemaSync(): Promise<CatalogSchemaDiff> { return this.fail(); }

backend/src/catalog/service.ts

@@ -2,6 +2,7 @@ import { buildInstallationContract } from "../workspaces/contracts.js";
 import { resolveBinding } from "../workspaces/bindings.js";
 import type { WorkspaceRegistry } from "../workspaces/registry.js";
 import type { WorkspaceDescriptor } from "../workspaces/schema.js";
+import { discoverWorkspaceSecretRequirements } from "../workspaces/secret-requirements.js";
 import type { WorkspaceSecretStore } from "../workspaces/secret-store.js";
 import { createConcreteDiagnosticAdapters } from "../workspaces/diagnostics.js";
 import { CatalogOperationCoordinator } from "./operation-coordinator.js";
@@ -25,6 +26,21 @@ export interface CatalogListItem extends Omit<WorkspaceDatabase, "id"> {
   workspaceName: string;
   workspaceDescription?: string;
   workspaceAvailable: boolean;
+  workspaceRevision: { commit: string; blob: string } | null;
+  workspaceEvidence: {
+    sourceType: "filesystem" | "http" | "s3" | null;
+    state:
+      | "not_declared"
+      | "materialized_current_revision"
+      | "configuration_required"
+      | "configured_unverified"
+      | "workspace_unavailable";
+  };
+  runtimeBinding: {
+    transport: DatabaseBinding["transport"];
+    configurationState: "ready" | "configuration_required";
+    sessionTransportSupported: boolean;
+  } | null;
   configured: boolean;
   secrets: Record<CatalogSecretName, boolean>;
 }
@@ -68,6 +84,43 @@ function secretState(store: WorkspaceSecretStore, workspaceId: string): Record<C
   ))) as Record<CatalogSecretName, boolean>;
 }
 
+function workspaceRuntimeState(
+  store: WorkspaceSecretStore,
+  workspace: WorkspaceDescriptor,
+  secretRoots: readonly string[],
+): NonNullable<CatalogListItem["runtimeBinding"]> {
+  const requirements = discoverWorkspaceSecretRequirements(workspace, process.env);
+  const secretVariables = new Set(requirements.map(({ variable }) => variable));
+  const effective = resolveBinding(workspace, "DWH", process.env, secretRoots);
+  const configurationRequired = requirements.some(({ id, required }) => (
+    required && !store.has(workspace.workspace.id, id)
+  )) || effective.missing.some((variable) => !secretVariables.has(variable));
+  return {
+    transport: effective.transport,
+    configurationState: configurationRequired ? "configuration_required" : "ready",
+    sessionTransportSupported: effective.transport !== "ssh_tunnel",
+  };
+}
+
+function workspaceEvidenceState(
+  store: WorkspaceSecretStore,
+  workspace: WorkspaceDescriptor,
+): CatalogListItem["workspaceEvidence"] {
+  const source = workspace.evidence?.source;
+  if (!source) return { sourceType: null, state: "not_declared" };
+  if (source.type === "filesystem") {
+    return { sourceType: source.type, state: "materialized_current_revision" };
+  }
+  const configurationRequired = discoverWorkspaceSecretRequirements(workspace, process.env)
+    .some(({ connector, id, required }) => (
+      connector === "evidence" && required && !store.has(workspace.workspace.id, id)
+    ));
+  return {
+    sourceType: source.type,
+    state: configurationRequired ? "configuration_required" : "configured_unverified",
+  };
+}
+
 export class CatalogService {
   private readonly adapters = createConcreteDiagnosticAdapters();
 
@@ -91,7 +144,8 @@ export class CatalogService {
     const byWorkspace = new Map(configured.map((database) => [database.workspaceId, database]));
     const active = await Promise.all(workspaces.map(async (entry) => {
       const database = byWorkspace.get(entry.id);
-      const { workspace } = await this.registry.read(entry.id);
+      const { workspace } = await this.registry.readPinned(entry.id, entry.revision.commit);
+      const runtimeDatabaseBinding = yamlBinding(workspace, this.secretRoots);
       const base = database ?? {
         workspaceId: entry.id,
         engine: "postgres" as const,
@@ -100,7 +154,7 @@ export class CatalogService {
         version: 0,
         createdAt: "",
         updatedAt: "",
-        binding: yamlBinding(workspace, this.secretRoots),
+        binding: runtimeDatabaseBinding,
         connectionStatus: "untested" as const,
       };
       return {
@@ -108,6 +162,12 @@ export class CatalogService {
         workspaceName: entry.name,
         workspaceDescription: entry.description,
         workspaceAvailable: true,
+        workspaceRevision: {
+          commit: entry.revision.commit,
+          blob: entry.revision.blob,
+        },
+        workspaceEvidence: workspaceEvidenceState(this.secretStore, workspace),
+        runtimeBinding: workspaceRuntimeState(this.secretStore, workspace, this.secretRoots),
         configured: database !== undefined,
         secrets: secretState(this.secretStore, entry.id),
       };
@@ -120,6 +180,9 @@ export class CatalogService {
         workspaceName: database.workspaceId,
         workspaceDescription: "Workspace is no longer present in the repository catalog.",
         workspaceAvailable: false,
+        workspaceRevision: null,
+        workspaceEvidence: { sourceType: null, state: "workspace_unavailable" },
+        runtimeBinding: null,
         configured: true,
         secrets: secretState(this.secretStore, database.workspaceId),
       }));

backend/src/catalog/types.ts

@@ -128,7 +128,7 @@ export interface CatalogRelationshipColumn {
   targetColumnName: string;
 }
 
-export interface CatalogRelationship {
+export interface CatalogPhysicalRelationship {
   id: string;
   databaseId: string;
   constraintName: string;
@@ -145,6 +145,52 @@ export interface CatalogRelationship {
   lastSyncedAt: string | null;
   createdAt: string;
   updatedAt: string;
+  origin: "physical";
+  status: "active";
+}
+
+export interface CatalogLogicalRelationship {
+  id: string;
+  databaseId: string;
+  constraintName: null;
+  sourceTableId: string;
+  sourceTableName: string;
+  targetTableId: string;
+  targetTableName: string;
+  updateRule: null;
+  deleteRule: null;
+  deferrable: false;
+  initiallyDeferred: false;
+  columns: [CatalogRelationshipColumn];
+  lastSyncedDatabaseVersion: null;
+  lastSyncedAt: null;
+  createdAt: string;
+  updatedAt: string;
+  origin: "generated" | "manual";
+  status: "active" | "excluded";
+}
+
+export type CatalogRelationship = CatalogPhysicalRelationship | CatalogLogicalRelationship;
+
+export interface CatalogLogicalRelationshipEndpoint {
+  columnId: string;
+  columnName: string;
+  tableId: string;
+  tableName: string;
+  dataType: string;
+  primaryKeyPosition: number | null;
+  tablePrimaryKeyColumnCount: number;
+}
+
+export interface CatalogLogicalRelationshipContext {
+  endpoints: CatalogLogicalRelationshipEndpoint[];
+  physicalPairs: Array<{ sourceColumnId: string; targetColumnId: string }>;
+  logicalRelationships: CatalogLogicalRelationship[];
+}
+
+export interface CatalogLogicalRelationshipCandidate {
+  sourceColumnId: string;
+  targetColumnId: string;
 }
 
 export type CatalogDatabaseMetadataDeleteTarget = "tables" | "relationships";
@@ -456,7 +502,25 @@ export interface CatalogRepository {
     runId: string,
     afterSequence?: number,
   ): Promise<SensitiveDataSuggestionEvent[]>;
-  listRelationships(databaseId: string): Promise<CatalogRelationship[]>;
+  listRelationships(databaseId: string): Promise<CatalogPhysicalRelationship[]>;
+  listLogicalRelationships(databaseId: string): Promise<CatalogLogicalRelationship[]>;
+  getLogicalRelationshipContext(databaseId: string): Promise<CatalogLogicalRelationshipContext | undefined>;
+  insertLogicalRelationship(
+    databaseId: string,
+    sourceColumnId: string,
+    targetColumnId: string,
+    generated: boolean,
+  ): Promise<CatalogLogicalRelationship | undefined>;
+  insertGeneratedLogicalRelationships(
+    databaseId: string,
+    candidates: readonly CatalogLogicalRelationshipCandidate[],
+  ): Promise<number>;
+  setLogicalRelationshipStatus(
+    databaseId: string,
+    relationshipId: string,
+    status: CatalogLogicalRelationship["status"],
+  ): Promise<CatalogLogicalRelationship | undefined>;
+  deleteLogicalRelationship(databaseId: string, relationshipId: string): Promise<boolean>;
   deleteDatabaseMetadata(
     databaseIds: readonly string[],
     target: CatalogDatabaseMetadataDeleteTarget,