From 076c9742c59941194826beff2780c24451e613bf Mon Sep 17 00:00:00 2001 From: Codex Date: Tue, 1 Sep 2026 14:46:55 +0200 Subject: [PATCH] feat: consolidate database management work Add catalog-owned logical relationships and runtime snapshots, extend the database-management UI and validation coverage, and document the updated operational workflow. Keep active sensitive-generation status in a tooltip and indicator, and update the layout E2E to follow the history action in its new database-scoped location. --- CONTEXT.md | 36 +- DESIGN.md | 44 +- PROJECT_STATE.md | 36 +- backend/src/app.ts | 20 + .../effective-relationship-snapshot.ts | 93 ++ .../catalog/logical-relationship-service.ts | 260 +++++ backend/src/catalog/memory-repository.ts | 153 ++- backend/src/catalog/migrate.ts | 2 + .../008_catalog_logical_relationships.ts | 35 + backend/src/catalog/repository.ts | 180 ++- backend/src/catalog/service.ts | 67 +- backend/src/catalog/types.ts | 68 +- .../routes/catalog-logical-relationships.ts | 154 +++ backend/src/routes/catalog-schema.ts | 11 - backend/src/routes/sessions.ts | 43 +- backend/src/tht/tht-runner.ts | 39 +- .../src/workspaces/runtime-config-lease.ts | 27 +- backend/src/workspaces/runtime-renderer.ts | 2 + backend/test/catalog-databases-routes.test.ts | 67 +- ...talog-logical-relationship-service.test.ts | 207 ++++ .../catalog-repository.integration.test.ts | 65 ++ backend/test/catalog-schema-routes.test.ts | 123 ++ backend/test/catalog-tables-routes.test.ts | 1 + .../effective-relationship-snapshot.test.ts | 196 ++++ backend/test/routes-sessions.test.ts | 57 + .../test/workspace-runtime-handoff.test.ts | 19 + ...g-as-the-logical-relationship-authority.md | 36 + docs/operations/database-management.md | 19 +- docs/operations/docker-refresh.md | 43 + ...026-08-26-metadata-catalog-from-thothai.md | 13 +- .../2026-08-31-browser-erd-library-options.md | 299 +++++ ...ationship-management-thothai-to-thothii.md | 246 ++++ ...-metadata-privacy-description-test-plan.md | 428 +++++++ docs/testing/evidence-lifecycle-test-plan.md | 1030 +++++++++++++++++ .../e2e/database-management-layout.spec.ts | 283 ++++- frontend/src/api/catalog-databases.test.ts | 106 +- frontend/src/api/catalog-databases.ts | 64 +- frontend/src/api/client.test.ts | 9 + frontend/src/api/client.ts | 12 + frontend/src/components/ui/button.tsx | 2 + frontend/src/index.css | 10 + frontend/src/shell/AppShell.auth.test.tsx | 53 +- .../AppShell.database-management.test.tsx | 35 +- .../src/shell/AppShell.new-session.test.tsx | 24 +- .../src/shell/AppShell.session-mgmt.test.tsx | 84 +- frontend/src/shell/AppShell.tsx | 283 +++-- .../src/shell/DatabaseManagementPage.test.tsx | 199 +++- frontend/src/shell/DatabaseManagementPage.tsx | 186 ++- .../database-management/CatalogSyncDrawer.tsx | 1 + .../database-management/DatabaseColumns.tsx | 11 +- .../DatabaseFleetQueryErrors.test.tsx | 3 + .../database-management/DatabaseForm.tsx | 59 +- .../database-management/DatabaseGrid.tsx | 267 ++++- .../DatabaseRelationships.test.tsx | 379 ++++++ .../DatabaseRelationships.tsx | 439 ++++++- .../database-management/DatabaseTables.tsx | 23 +- .../DescriptionGenerationDrawer.test.tsx | 27 + .../DescriptionGenerationDrawer.tsx | 23 +- .../FleetActionSelector.typography.test.ts | 30 + .../database-management/FleetLedgerShell.css | 67 +- .../database-management/FleetLedgerShell.tsx | 2 +- .../HistoryActionPlacement.test.ts | 22 + .../MetadataGenerationModelSelector.tsx | 39 +- .../RecentRunsSuccessStyle.test.ts | 27 + .../SensitiveDataSuggestionHistoryDrawer.tsx | 21 +- .../src/shell/database-management/model.ts | 3 +- harness/tests/test_effective_relationships.py | 403 +++++++ harness/tht/cli/schema_cmd.py | 69 +- harness/tht/cli/search_cmd.py | 17 +- harness/tht/config.py | 4 + harness/tht/mschema/context.py | 133 +++ harness/tht/mschema/render.py | 37 +- mkdocs.yml | 1 + 73 files changed, 6966 insertions(+), 610 deletions(-) create mode 100644 backend/src/catalog/effective-relationship-snapshot.ts create mode 100644 backend/src/catalog/logical-relationship-service.ts create mode 100644 backend/src/catalog/migrations/008_catalog_logical_relationships.ts create mode 100644 backend/src/routes/catalog-logical-relationships.ts create mode 100644 backend/test/catalog-logical-relationship-service.test.ts create mode 100644 backend/test/effective-relationship-snapshot.test.ts create mode 100644 docs/adr/0012-use-the-catalog-as-the-logical-relationship-authority.md create mode 100644 docs/operations/docker-refresh.md create mode 100644 docs/research/2026-08-31-browser-erd-library-options.md create mode 100644 docs/research/2026-08-31-relationship-management-thothai-to-thothii.md create mode 100644 docs/testing/2026-08-31-metadata-privacy-description-test-plan.md create mode 100644 docs/testing/evidence-lifecycle-test-plan.md create mode 100644 frontend/src/shell/database-management/DatabaseRelationships.test.tsx create mode 100644 frontend/src/shell/database-management/FleetActionSelector.typography.test.ts create mode 100644 frontend/src/shell/database-management/HistoryActionPlacement.test.ts create mode 100644 frontend/src/shell/database-management/RecentRunsSuccessStyle.test.ts create mode 100644 harness/tests/test_effective_relationships.py create mode 100644 harness/tht/mschema/context.py diff --git a/CONTEXT.md b/CONTEXT.md index 5cb94c78..898aaa04 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -296,8 +296,40 @@ Metadata Catalog. Non è creata o modificata manualmente, ma può essere rimossa Metadata Cleanup. _Avoid_: denormalized FK, relationship string -**Logical Relationship** — Una relazione semantica curata o inferita che non corrisponde -necessariamente a un vincolo fisico. Ha ownership e lifecycle distinti da Catalog Relationship. +**Logical Relationship** — Una relazione modificabile fra due Catalog Column che non corrisponde +necessariamente a un vincolo fisico. Può essere Generated o Manual e rimane distinta dalla Catalog +Relationship osservata nel database. + +**Generated Relationship** — Una Logical Relationship ricavata dai nomi delle colonne, dalle +primary key e dalla compatibilità dei tipi mediante regole deterministiche, senza LLM, embedding o +campionamento dei dati. Una ricostruzione non riattiva una Generated Relationship cancellata +logicamente, ma può ricrearne una cancellata fisicamente. + +**Manual Relationship** — Una Logical Relationship aggiunta dall'utente. La ricostruzione delle +Generated Relationship non la modifica. + +**Logical Relationship Deletion** — L'esclusione persistente di una Logical Relationship che ne +conserva l'identità per impedirne la ricreazione automatica finché esistono entrambe le Catalog +Column alle quali è collegata. + +**Permanent Relationship Deletion** — La rimozione completa di una Logical Relationship. Una +ricostruzione successiva può ricrearla quando soddisfa nuovamente le regole di inferenza. Anche il +cleanup distruttivo di una tabella o colonna endpoint rimuove permanentemente le relative esclusioni. + +**Relationship Reconstruction** — L'operazione amministrativa esplicita che scopre e aggiunge le +Generated Relationship mancanti. Conserva le Manual Relationship e le relationship già presenti e +non riattiva quelle cancellate logicamente. + +**Relationship Restore** — La riattivazione esplicita di una Logical Relationship cancellata +logicamente. + +**Effective Relationship Map** — La vista unificata delle Catalog Relationship fisiche e delle +Logical Relationship, con origine e stato espliciti. È l'interfaccia usata dall'amministrazione e +dalla comprensione dello schema, non un ulteriore modello persistito. + +**Effective Relationship Snapshot** — La proiezione runtime immutabile delle relationship attive +contenute nell'Effective Relationship Map. È derivata dal Metadata Catalog per una singola sessione +e viene eliminata insieme alla relativa configurazione runtime. **Description** — Il testo curato e consolidato che descrive una Catalog Table o Catalog Column per gli usi downstream. diff --git a/DESIGN.md b/DESIGN.md index e466ce2c..29c99d12 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -12,6 +12,10 @@ colors: muted-graphite: "oklch(51.33% 0.0088 345.6)" quiet-border: "oklch(90.93% 0.0035 354.7)" success-mint: "oklch(75.77% 0.1581 165)" + navigation-active: "oklch(92.5% 0.052 23.2)" + navigation-active-hover: "oklch(89.5% 0.071 23.2)" + navigation-active-foreground: "oklch(36.5% 0.11 23.2)" + navigation-active-border: "oklch(60% 0.135 23.2)" warning-amber: "oklch(85.23% 0.1386 78.9)" information-blue: "oklch(70.35% 0.1128 221.3)" typography: @@ -152,8 +156,8 @@ frontend uses OKLCH tokens directly. ### Primary -- **Instrument Red** (`instrument-red`): primary actions, current selection, focus identity, and - destructive meaning where the context already makes the action explicit. +- **Instrument Red** (`instrument-red`): primary actions, focus identity, and destructive meaning + where the context already makes the action explicit. - **Instrument Red Pressed** (`instrument-red-hover`): hover and active emphasis for the primary action family. @@ -170,6 +174,9 @@ frontend uses OKLCH tokens directly. ### Semantic - **Success Mint** (`success-mint`): completed and ready states. +- **Navigation Active** (`navigation-active`): the one application surface currently in the + foreground. It shares Instrument Red's hue but uses a lighter, lower-chroma fill, so location is + visible without carrying the full weight of a primary action. - **Warning Amber** (`warning-amber`): waiting, attention, and in-progress states. - **Information Blue** (`information-blue`): informational state when red would imply action. @@ -272,15 +279,44 @@ default, hover, focus, active, disabled, loading, and error behavior where those - **Focus:** three-pixel Instrument Red ring with a clear border shift. - **Error / Disabled:** errors combine destructive color with explanatory text; disabled controls retain readable contrast and use 50 percent opacity. +- **Metadata catalog model:** Database Management keeps one compact, installation-level + metadata-generation LLM selector in the application header. The selection persists across + database, table, column, and relationship views; when no usable profile is configured, the + disabled control explains: “No metadata-generation LLM model is configured for this installation.” ### Navigation - **Style:** compact session rows use `8px` corners and restrained vertical padding. -- **Default / Hover / Active:** transparent at rest, Sunken Surface on hover, and the same surface - with stronger text weight when active. +- **Default / Hover / Active:** porcelain at rest, Sunken Surface on hover, and a muted Navigation + Active red with a defined border when current. Exactly one top-level navigation control is current. +- **Administrative controls:** the admin-only Administration accordion groups Database management, + a structural divider, Workspace management, and Pi management in that order. Its trigger exposes + expanded state and starts collapsed by default, while non-admin users do not receive the accordion + or its navigation actions. - **Responsive:** collapse navigation structurally at the application breakpoint. Do not shrink labels into illegibility. +### Tabs + +- **Shape:** compact label tabs sit on a shared baseline with rounded top corners and a two-pixel + lower edge. Inactive labels retain a complete Quiet Border and Porcelain Card surface, so every + label reads as a tab before interaction; hover feedback reinforces clickability. +- **Current:** the selected tab uses the muted Navigation Active red for its fill, text, and defined border. + It must expose `aria-selected`, participate in a labelled `tablist`/`tabpanel`, and be the only + tab in the roving keyboard tab order. +- **Keyboard:** Left/Right move between adjacent tabs with wrapping; Home/End select the first or + last tab. + +### Tooltips + +- **Row actions:** icon-action tooltips open three pixels below the trigger and align to its trailing + edge, so they never cover the icon row. They use a dark slate surface, porcelain text, and a + defined border rather than the light popover treatment. +- **Interaction:** tooltip layers never receive pointer events. They appear on hover and keyboard + focus with a short ease-out transition, while the icon button keeps its complete accessible name. +- **Scope:** this treatment is shared by database, table, column, and relationship row actions. + Toolbar and navigation hints may use separate collision-aware placement. + ### Curated Evidence Documents Curated evidence follows a fixed reading order: title, compact type and purpose summary, scope, diff --git a/PROJECT_STATE.md b/PROJECT_STATE.md index 5007fddb..b7b37cf7 100644 --- a/PROJECT_STATE.md +++ b/PROJECT_STATE.md @@ -1,6 +1,6 @@ # ThothII — Project State -Last updated: 2026-08-27. +Last updated: 2026-08-31. This file is the short operational snapshot. Stable commands and the architecture mental model live in `AGENTS.md`; current design and runtime contracts live under `docs/architecture/`, @@ -90,6 +90,18 @@ relationships. Curated and generated descriptions are editable; generated descri and Database Management can generate or consolidate them for selected tables, selected columns, all targets, or only targets whose Generated Description is missing. +Relationship Management is now reachable directly from each configured Fleet database. One +Relationship Map shows read-only Physical Relationships together with Generated and Manual Logical +Relationships, with Active, Excluded, and All filters. Administrators can add a single-column +relationship, run deterministic name/PK/type inference, exclude or restore a logical relationship, +or delete it permanently. Exclusion retains a tombstone that a rebuild cannot reactivate; permanent +deletion allows a later rebuild to infer the same endpoints again. Inference uses no LLM, embedding, +or source values. It supports normalized table-qualified names, unique non-generic PK names, +composite-PK source columns, and the `*time_key -> dim_time.` warehouse convention while +ignoring bare generic names. Explicit table/column metadata cleanup remains a destructive boundary: it removes +the attached logical relationships and exclusions and requires a full schema synchronization before +inference or runtime publication can continue. + The previous Database Management renderer remains a temporary comparison fallback for development and staging only: `?db-ui=legacy` is honored in Vite development or when `VITE_DB_MANAGEMENT_LEGACY=true`; it is not a production presentation. The standalone Fleet Ledger @@ -115,13 +127,15 @@ SSH is not yet enabled for NL→SQL session runtime. The catalog runs in the internal `catalog-db` PostgreSQL service. Kysely migrations are an explicit one-shot `catalog-migrate` operation; `scripts/run-stack.sh` runs it before local startup. Runtime -sessions still consume the existing workspace configuration in this slice: database-management -records do not yet change the NL→SQL handoff. The accepted design is recorded in +sessions now consume the Catalog's active effective relationship map through an immutable JSON +snapshot tied to the runtime-config lease. The harness uses that snapshot as its exclusive +relationship source while retaining Git-pinned annotations for descriptive metadata; legacy +runtimes without a snapshot keep the previous merge behavior. The accepted design is recorded in `docs/plans/2026-08-26-metadata-catalog-from-thothai.md`, the snapshot contract under -`docs/contracts/`, and ADRs 0001–0011. +`docs/contracts/`, and ADRs 0001–0012. -Semantic aliases, value descriptions, synonyms, concepts, and logical relationships remain -deferred to their dedicated slices. +Semantic aliases, value descriptions, synonyms, and concepts remain deferred to their dedicated +slices. AI Description Generation uses the catalog's human-owned Sensitive Data Flag. The flag defaults to `false`, including for newly synchronized columns. An administrator may request an AI proposal based @@ -158,11 +172,11 @@ credential. The Python client supplies only its fixed non-secret compatibility p The AritmoLab entry also sets `disableThinking: true`, mapped to the endpoint's chat-template flag, because its default reasoning prose would violate the worker's exact JSON response contract. -Integration of the completed metadata catalog with core schema-linking is explicitly deferred -until the database, table, column, relationship, and synchronization slices are complete. At that -point the next required design gate is to compare the catalog snapshot with the current DWH -preprocessing/schema-linking contracts and plan the cutover; this follow-up must not be treated as -optional cleanup or silently omitted. +Logical relationship integration with core schema-linking is complete: session creation and resume +materialize the active physical/generated/manual map, retrieval-pack generation and Pi receive the +same runtime config, and snapshot validation fails closed on a declared missing, invalid, or orphaned +endpoint. Broader publication of other Catalog metadata to schema-linking remains a separate future +slice. **Deferred follow-up — Sensitive Data Policy in schema-linking.** The policy is first delivered and tested in catalog description generation. Its enforcement for core schema-linking remains diff --git a/backend/src/app.ts b/backend/src/app.ts index 1b8dc64a..01c5c7e9 100644 --- a/backend/src/app.ts +++ b/backend/src/app.ts @@ -63,6 +63,9 @@ import { type DescriptionSourceSampler, } from "./catalog/description-source-sampler.js"; import { catalogDescriptionGenerationRoutes } from "./routes/catalog-description-generation.js"; +import { CatalogLogicalRelationshipService } from "./catalog/logical-relationship-service.js"; +import { catalogLogicalRelationshipRoutes } from "./routes/catalog-logical-relationships.js"; +import { EffectiveRelationshipSnapshotProvider } from "./catalog/effective-relationship-snapshot.js"; export interface BuildAppDeps { thtRunner?: ThtRunner; @@ -79,6 +82,8 @@ export interface BuildAppDeps { catalogService?: CatalogService; catalogPostgresAccess?: CatalogPostgresAccess; catalogTableService?: CatalogTableService; + catalogLogicalRelationshipService?: CatalogLogicalRelationshipService; + effectiveRelationshipSnapshotProvider?: EffectiveRelationshipSnapshotProvider; catalogSchemaIntrospector?: CatalogSchemaIntrospector; catalogSyncWorker?: CatalogSyncWorker; catalogOperationCoordinator?: CatalogOperationCoordinator; @@ -200,6 +205,16 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc catalogOperationCoordinator, ); const catalogTableService = deps?.catalogTableService ?? new CatalogTableService(catalogRepository); + const catalogLogicalRelationshipService = deps?.catalogLogicalRelationshipService + ?? new CatalogLogicalRelationshipService(catalogRepository); + const effectiveRelationships = deps?.effectiveRelationshipSnapshotProvider + ?? (config.catalogDatabase === undefined + ? undefined + : new EffectiveRelationshipSnapshotProvider( + catalogRepository, + catalogLogicalRelationshipService, + catalogOperationCoordinator, + )); const catalogSchemaIntrospector = deps?.catalogSchemaIntrospector ?? new ConcreteCatalogSchemaIntrospector( catalogPostgresAccess, workspaceSecretStore, @@ -394,6 +409,7 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc legacyWorkspaceMode: config.legacyWorkspaceMode, workspaceRuntimeSupport, maintenanceBarrier, + effectiveRelationships, }); app.post("/internal/maintenance/activate", async (req, reply) => { try { @@ -442,6 +458,10 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc worker: catalogSyncWorker, operations: catalogOperationCoordinator, }); + catalogLogicalRelationshipRoutes(app, { + service: catalogLogicalRelationshipService, + operations: catalogOperationCoordinator, + }); catalogDescriptionConsolidationRoutes(app, { repository: catalogRepository, operations: catalogOperationCoordinator, diff --git a/backend/src/catalog/effective-relationship-snapshot.ts b/backend/src/catalog/effective-relationship-snapshot.ts new file mode 100644 index 00000000..35816281 --- /dev/null +++ b/backend/src/catalog/effective-relationship-snapshot.ts @@ -0,0 +1,93 @@ +import type { CatalogRelationship, CatalogRepository } from "./types.js"; + +export interface EffectiveRelationshipSnapshotReader { + list(databaseId: string): Promise; +} + +export interface EffectiveRelationshipSnapshotCoordinator { + run(databaseId: string, operation: () => Promise): Promise; +} + +export class EffectiveRelationshipSnapshotStaleError extends Error {} + +interface EffectiveRelationship { + sourceTable: string; + sourceColumns: string[]; + targetTable: string; + targetColumns: string[]; + origin: CatalogRelationship["origin"]; +} + +interface EffectiveRelationshipSnapshot { + schemaVersion: 1; + workspaceId: string; + relationships: EffectiveRelationship[]; +} + +const originRank: Record = { + physical: 0, + manual: 1, + generated: 2, +}; + +function endpointKey(relationship: EffectiveRelationship): string { + return [ + relationship.sourceTable, + relationship.sourceColumns.join("\u0000"), + relationship.targetTable, + relationship.targetColumns.join("\u0000"), + ].join("\u0001"); +} + +function compareRelationships(left: EffectiveRelationship, right: EffectiveRelationship): number { + return endpointKey(left).localeCompare(endpointKey(right)) + || originRank[left.origin] - originRank[right.origin]; +} + +/** + * Adapter from the mutable Catalog model to the immutable relationship contract consumed by the + * harness. The returned JSON is a deterministic projection, never an authored second store. + */ +export class EffectiveRelationshipSnapshotProvider { + constructor( + private readonly repository: Pick, + private readonly relationships: EffectiveRelationshipSnapshotReader, + private readonly operations: EffectiveRelationshipSnapshotCoordinator, + ) {} + + async render(workspaceId: string): Promise { + const database = await this.repository.getByWorkspace(workspaceId); + if (!database) return undefined; + return await this.operations.run(database.id, async () => { + const current = await this.repository.get(database.id); + if (!current || current.schemaSyncedVersion !== current.version) { + throw new EffectiveRelationshipSnapshotStaleError( + "effective relationship snapshot requires a current full schema synchronization", + ); + } + const projected = (await this.relationships.list(database.id)) + .filter((relationship) => relationship.status === "active") + .map((relationship): EffectiveRelationship => ({ + sourceTable: relationship.sourceTableName, + sourceColumns: relationship.columns.map((column) => column.sourceColumnName), + targetTable: relationship.targetTableName, + targetColumns: relationship.columns.map((column) => column.targetColumnName), + origin: relationship.origin, + })) + .sort(compareRelationships); + + const seen = new Set(); + const snapshot: EffectiveRelationshipSnapshot = { + schemaVersion: 1, + workspaceId, + relationships: projected.filter((relationship) => { + const key = endpointKey(relationship); + if (seen.has(key)) return false; + seen.add(key); + return true; + }), + }; + return `${JSON.stringify(snapshot, null, 2)}\n`; + }); + } +} diff --git a/backend/src/catalog/logical-relationship-service.ts b/backend/src/catalog/logical-relationship-service.ts new file mode 100644 index 00000000..671fa7d4 --- /dev/null +++ b/backend/src/catalog/logical-relationship-service.ts @@ -0,0 +1,260 @@ +import type { + CatalogLogicalRelationship, + CatalogLogicalRelationshipCandidate, + CatalogLogicalRelationshipContext, + CatalogLogicalRelationshipEndpoint, + CatalogRelationship, + CatalogRepository, +} from "./types.js"; + +export class LogicalRelationshipDatabaseNotFoundError extends Error {} +export class LogicalRelationshipDuplicateError extends Error {} +export class LogicalRelationshipNotFoundError extends Error {} +export class LogicalRelationshipReadOnlyError extends Error {} +export class LogicalRelationshipSchemaStaleError extends Error {} +export class LogicalRelationshipTargetNotUniqueError extends Error {} +export class LogicalRelationshipTypeIncompatibleError extends Error {} +export class LogicalRelationshipColumnNotFoundError extends Error { + constructor(readonly field: "sourceColumnId" | "targetColumnId") { + super(`Catalog column '${field}' was not found`); + } +} + +export interface RebuildGeneratedRelationshipsResult { + added: number; + alreadyPresent: number; + excluded: number; + ambiguous: number; +} + +function identifierTokens(value: string): string[] { + return value + .replace(/([a-z0-9])([A-Z])/g, "$1_$2") + .toLowerCase() + .split(/[^a-z0-9]+/) + .filter(Boolean); +} + +function singularWord(value: string): string { + if (value.length > 4 && value.endsWith("ies")) return `${value.slice(0, -3)}y`; + if (value.length > 4 && /(ches|shes|xes|zes|ses)$/.test(value)) return value.slice(0, -2); + if (value.length > 3 && value.endsWith("s") && !/(ss|us)$/.test(value)) return value.slice(0, -1); + return value; +} + +function tableAliases(tableName: string): string[] { + const tokens = identifierTokens(tableName); + if (tokens.length === 0) return []; + const normalized = tokens.join("_"); + const singular = [...tokens]; + singular[singular.length - 1] = singularWord(singular[singular.length - 1]); + return [...new Set([normalized, singular.join("_")])]; +} + +const GENERIC_PRIMARY_KEY_NAMES = new Set(["id", "key", "code", "pk"]); + +function nameMatches( + source: CatalogLogicalRelationshipEndpoint, + target: CatalogLogicalRelationshipEndpoint, +): boolean { + const sourceName = identifierTokens(source.columnName).join("_"); + const targetName = identifierTokens(target.columnName).join("_"); + if (!sourceName || !targetName) return false; + const expected = new Set(); + if (!GENERIC_PRIMARY_KEY_NAMES.has(targetName)) expected.add(targetName); + for (const alias of tableAliases(target.tableName)) { + expected.add(`${alias}_${targetName}`); + expected.add(`${alias.replaceAll("_", "")}${targetName.replaceAll("_", "")}`); + if (targetName === "id" || targetName === "pk") expected.add(alias); + } + return expected.has(sourceName); +} + +function canonicalDataType(value: string): string { + const normalized = value.trim().toLowerCase().replace(/\s+/g, " "); + const arraySuffix = normalized.endsWith("[]") ? "[]" : ""; + const base = arraySuffix ? normalized.slice(0, -2) : normalized; + const withoutModifier = base.replace(/\([^)]*\)/g, "").trim(); + const aliases: Record = { + int2: "smallint", + smallserial: "smallint", + int4: "integer", + int: "integer", + serial: "integer", + int8: "bigint", + bigserial: "bigint", + decimal: "numeric", + varchar: "text", + "character varying": "text", + bool: "boolean", + "timestamp without time zone": "timestamp", + "timestamp with time zone": "timestamptz", + "time without time zone": "time", + "time with time zone": "timetz", + }; + return `${aliases[withoutModifier] ?? withoutModifier}${arraySuffix}`; +} + +function typesCompatible(left: string, right: string): boolean { + return canonicalDataType(left) === canonicalDataType(right); +} + +function pairKey(sourceColumnId: string, targetColumnId: string): string { + return `${sourceColumnId}\u0000${targetColumnId}`; +} + +function relationshipPair(relationship: CatalogLogicalRelationship): CatalogLogicalRelationshipCandidate { + return { + sourceColumnId: relationship.columns[0].sourceColumnId, + targetColumnId: relationship.columns[0].targetColumnId, + }; +} + +function relationshipSortKey(relationship: CatalogRelationship): string { + const sourceColumns = relationship.columns.map((column) => column.sourceColumnName).join(","); + const targetColumns = relationship.columns.map((column) => column.targetColumnName).join(","); + return [ + relationship.sourceTableName, + sourceColumns, + relationship.targetTableName, + targetColumns, + relationship.origin, + ].join("\u0000"); +} + +export class CatalogLogicalRelationshipService { + constructor(private readonly repository: CatalogRepository) {} + + async list(databaseId: string): Promise { + if (!(await this.repository.get(databaseId))) throw new LogicalRelationshipDatabaseNotFoundError(); + const relationships: CatalogRelationship[] = [ + ...await this.repository.listRelationships(databaseId), + ...await this.repository.listLogicalRelationships(databaseId), + ]; + return relationships.sort((left, right) => relationshipSortKey(left).localeCompare(relationshipSortKey(right))); + } + + async addManual( + databaseId: string, + sourceColumnId: string, + targetColumnId: string, + ): Promise { + const context = await this.requiredContext(databaseId); + const source = context.endpoints.find((endpoint) => endpoint.columnId === sourceColumnId); + if (!source) throw new LogicalRelationshipColumnNotFoundError("sourceColumnId"); + const target = context.endpoints.find((endpoint) => endpoint.columnId === targetColumnId); + if (!target) throw new LogicalRelationshipColumnNotFoundError("targetColumnId"); + if (sourceColumnId === targetColumnId + || target.primaryKeyPosition === null + || target.tablePrimaryKeyColumnCount !== 1) { + throw new LogicalRelationshipTargetNotUniqueError(); + } + if (!typesCompatible(source.dataType, target.dataType)) { + throw new LogicalRelationshipTypeIncompatibleError(); + } + const key = pairKey(sourceColumnId, targetColumnId); + if (context.physicalPairs.some((pair) => pairKey(pair.sourceColumnId, pair.targetColumnId) === key) + || context.logicalRelationships.some((relationship) => { + const pair = relationshipPair(relationship); + return pairKey(pair.sourceColumnId, pair.targetColumnId) === key; + })) throw new LogicalRelationshipDuplicateError(); + const created = await this.repository.insertLogicalRelationship( + databaseId, + sourceColumnId, + targetColumnId, + false, + ); + if (!created) throw new LogicalRelationshipDuplicateError(); + return created; + } + + async rebuildGenerated(databaseId: string): Promise { + const context = await this.requiredContext(databaseId); + const targets = context.endpoints.filter((endpoint) => ( + endpoint.primaryKeyPosition !== null && endpoint.tablePrimaryKeyColumnCount === 1 + )); + const physical = new Set(context.physicalPairs.map((pair) => pairKey(pair.sourceColumnId, pair.targetColumnId))); + const active = new Set(); + const excluded = new Set(); + for (const relationship of context.logicalRelationships) { + const pair = relationshipPair(relationship); + (relationship.status === "excluded" ? excluded : active) + .add(pairKey(pair.sourceColumnId, pair.targetColumnId)); + } + + const pending: CatalogLogicalRelationshipCandidate[] = []; + let alreadyPresent = 0; + let excludedCount = 0; + let ambiguous = 0; + const dimTimeTargets = targets.filter((target) => ( + identifierTokens(target.tableName).join("_") === "dim_time" + )); + const sources = context.endpoints.filter((endpoint) => ( + endpoint.primaryKeyPosition === null || endpoint.tablePrimaryKeyColumnCount > 1 + )); + for (const source of sources) { + const sourceName = identifierTokens(source.columnName).join("_"); + const isTimeKey = sourceName.endsWith("time_key") + && identifierTokens(source.tableName).join("_") !== "dim_time"; + const candidates = isTimeKey ? dimTimeTargets : targets; + const matches = candidates.filter((target) => ( + target.columnId !== source.columnId + && typesCompatible(source.dataType, target.dataType) + && (isTimeKey || nameMatches(source, target)) + )); + if (matches.length > 1) { + ambiguous += 1; + continue; + } + if (matches.length === 0) continue; + const candidate = { sourceColumnId: source.columnId, targetColumnId: matches[0].columnId }; + const key = pairKey(candidate.sourceColumnId, candidate.targetColumnId); + if (physical.has(key) || active.has(key)) { + alreadyPresent += 1; + } else if (excluded.has(key)) { + excludedCount += 1; + } else { + pending.push(candidate); + } + } + const added = await this.repository.insertGeneratedLogicalRelationships(databaseId, pending); + alreadyPresent += pending.length - added; + return { added, alreadyPresent, excluded: excludedCount, ambiguous }; + } + + async setStatus( + databaseId: string, + relationshipId: string, + status: CatalogLogicalRelationship["status"], + ): Promise { + if (!(await this.repository.get(databaseId))) throw new LogicalRelationshipDatabaseNotFoundError(); + const updated = await this.repository.setLogicalRelationshipStatus(databaseId, relationshipId, status); + if (updated) return updated; + await this.assertNotPhysical(databaseId, relationshipId); + throw new LogicalRelationshipNotFoundError(); + } + + async deletePermanently(databaseId: string, relationshipId: string): Promise { + if (!(await this.repository.get(databaseId))) throw new LogicalRelationshipDatabaseNotFoundError(); + if (await this.repository.deleteLogicalRelationship(databaseId, relationshipId)) return; + await this.assertNotPhysical(databaseId, relationshipId); + throw new LogicalRelationshipNotFoundError(); + } + + private async requiredContext(databaseId: string): Promise { + const database = await this.repository.get(databaseId); + if (!database) throw new LogicalRelationshipDatabaseNotFoundError(); + if (database.schemaSyncedVersion !== database.version) { + throw new LogicalRelationshipSchemaStaleError(); + } + const context = await this.repository.getLogicalRelationshipContext(databaseId); + if (!context) throw new LogicalRelationshipDatabaseNotFoundError(); + return context; + } + + private async assertNotPhysical(databaseId: string, relationshipId: string): Promise { + if ((await this.repository.listRelationships(databaseId)).some((relationship) => relationship.id === relationshipId)) { + throw new LogicalRelationshipReadOnlyError(); + } + } +} diff --git a/backend/src/catalog/memory-repository.ts b/backend/src/catalog/memory-repository.ts index 40a14488..a373250f 100644 --- a/backend/src/catalog/memory-repository.ts +++ b/backend/src/catalog/memory-repository.ts @@ -14,7 +14,10 @@ import { type CatalogDatabaseMetadataDeleteTarget, type CatalogMetadataDeleteCounts, type CatalogMetrics, - type CatalogRelationship, + type CatalogLogicalRelationship, + type CatalogLogicalRelationshipCandidate, + type CatalogLogicalRelationshipContext, + type CatalogPhysicalRelationship, type CatalogSchemaDiff, type CatalogSyncCounts, type CatalogSyncEvent, @@ -49,7 +52,8 @@ export class MemoryCatalogRepository implements CatalogRepository { private readonly records = new Map(); private readonly tables = new Map(); private readonly columns = new Map(); - private readonly relationships = new Map(); + private readonly relationships = new Map(); + private readonly logicalRelationships = new Map(); private readonly descriptionGenerationRuns = new Map(); private readonly descriptionGenerationEvents = new Map(); private readonly sensitiveDataSuggestionRuns = new Map(); @@ -171,6 +175,9 @@ export class MemoryCatalogRepository implements CatalogRepository { for (const [relationshipId, relationship] of this.relationships) { if (relationship.databaseId === id) this.relationships.delete(relationshipId); } + for (const [relationshipId, relationship] of this.logicalRelationships) { + if (relationship.databaseId === id) this.logicalRelationships.delete(relationshipId); + } for (const [runId, run] of this.descriptionGenerationRuns) { if (run.databaseId !== id) continue; this.descriptionGenerationRuns.delete(runId); @@ -527,12 +534,138 @@ export class MemoryCatalogRepository implements CatalogRepository { .map((event) => structuredClone(event)); } - async listRelationships(databaseId: string): Promise { + async listRelationships(databaseId: string): Promise { return [...this.relationships.values()].filter((relationship) => relationship.databaseId === databaseId) .sort((a, b) => `${a.sourceTableName}.${a.constraintName}`.localeCompare(`${b.sourceTableName}.${b.constraintName}`)) .map((relationship) => structuredClone(relationship)); } + async listLogicalRelationships(databaseId: string): Promise { + return [...this.logicalRelationships.values()] + .filter((relationship) => relationship.databaseId === databaseId) + .sort((a, b) => { + const left = `${a.sourceTableName}.${a.columns[0].sourceColumnName}.${a.targetTableName}.${a.columns[0].targetColumnName}`; + const right = `${b.sourceTableName}.${b.columns[0].sourceColumnName}.${b.targetTableName}.${b.columns[0].targetColumnName}`; + return left.localeCompare(right); + }) + .map((relationship) => structuredClone(relationship)); + } + + async getLogicalRelationshipContext( + databaseId: string, + ): Promise { + if (!this.records.has(databaseId)) return undefined; + const tables = [...this.tables.values()].filter((table) => table.databaseId === databaseId); + const tableById = new Map(tables.map((table) => [table.id, table])); + const columns = [...this.columns.values()].filter((column) => tableById.has(column.tableId)); + const primaryKeyCounts = new Map(); + for (const column of columns) { + if (column.primaryKeyPosition !== null) { + primaryKeyCounts.set(column.tableId, (primaryKeyCounts.get(column.tableId) ?? 0) + 1); + } + } + return { + endpoints: columns.map((column) => ({ + columnId: column.id, + columnName: column.name, + tableId: column.tableId, + tableName: tableById.get(column.tableId)!.name, + dataType: column.dataType, + primaryKeyPosition: column.primaryKeyPosition, + tablePrimaryKeyColumnCount: primaryKeyCounts.get(column.tableId) ?? 0, + })), + physicalPairs: [...this.relationships.values()] + .filter((relationship) => relationship.databaseId === databaseId) + .flatMap((relationship) => relationship.columns.map((column) => ({ + sourceColumnId: column.sourceColumnId, + targetColumnId: column.targetColumnId, + }))), + logicalRelationships: await this.listLogicalRelationships(databaseId), + }; + } + + async insertLogicalRelationship( + databaseId: string, + sourceColumnId: string, + targetColumnId: string, + generated: boolean, + ): Promise { + if ([...this.logicalRelationships.values()].some((relationship) => ( + relationship.databaseId === databaseId + && relationship.columns[0].sourceColumnId === sourceColumnId + && relationship.columns[0].targetColumnId === targetColumnId + ))) return undefined; + const sourceColumn = this.columns.get(sourceColumnId); + const targetColumn = this.columns.get(targetColumnId); + const sourceTable = sourceColumn ? this.tables.get(sourceColumn.tableId) : undefined; + const targetTable = targetColumn ? this.tables.get(targetColumn.tableId) : undefined; + if (!sourceColumn || !targetColumn || !sourceTable || !targetTable + || sourceTable.databaseId !== databaseId || targetTable.databaseId !== databaseId + || sourceColumnId === targetColumnId) return undefined; + const now = new Date().toISOString(); + const relationship: CatalogLogicalRelationship = { + id: randomUUID(), + databaseId, + constraintName: null, + sourceTableId: sourceTable.id, + sourceTableName: sourceTable.name, + targetTableId: targetTable.id, + targetTableName: targetTable.name, + updateRule: null, + deleteRule: null, + deferrable: false, + initiallyDeferred: false, + columns: [{ + position: 1, + sourceColumnId, + sourceColumnName: sourceColumn.name, + targetColumnId, + targetColumnName: targetColumn.name, + }], + lastSyncedDatabaseVersion: null, + lastSyncedAt: null, + createdAt: now, + updatedAt: now, + origin: generated ? "generated" : "manual", + status: "active", + }; + this.logicalRelationships.set(relationship.id, relationship); + return structuredClone(relationship); + } + + async insertGeneratedLogicalRelationships( + databaseId: string, + candidates: readonly CatalogLogicalRelationshipCandidate[], + ): Promise { + let added = 0; + for (const candidate of candidates) { + if (await this.insertLogicalRelationship( + databaseId, + candidate.sourceColumnId, + candidate.targetColumnId, + true, + )) added += 1; + } + return added; + } + + async setLogicalRelationshipStatus( + databaseId: string, + relationshipId: string, + status: CatalogLogicalRelationship["status"], + ): Promise { + const current = this.logicalRelationships.get(relationshipId); + if (!current || current.databaseId !== databaseId) return undefined; + const updated = { ...current, status, updatedAt: new Date().toISOString() }; + this.logicalRelationships.set(relationshipId, updated); + return structuredClone(updated); + } + + async deleteLogicalRelationship(databaseId: string, relationshipId: string): Promise { + const current = this.logicalRelationships.get(relationshipId); + return Boolean(current?.databaseId === databaseId && this.logicalRelationships.delete(relationshipId)); + } + async deleteDatabaseMetadata( databaseIds: readonly string[], target: CatalogDatabaseMetadataDeleteTarget, @@ -574,6 +707,12 @@ export class MemoryCatalogRepository implements CatalogRepository { const columns = [...this.columns.values()].filter((column) => selected.has(column.tableId)); const deletedColumnIds = new Set(columns.map((column) => column.id)); for (const column of columns) this.columns.delete(column.id); + for (const relationship of [...this.logicalRelationships.values()]) { + const pair = relationship.columns[0]; + if (deletedColumnIds.has(pair.sourceColumnId) || deletedColumnIds.has(pair.targetColumnId)) { + this.logicalRelationships.delete(relationship.id); + } + } for (const [relationshipId, relationship] of this.relationships) { if (relationship.databaseId !== databaseId) continue; this.relationships.set(relationshipId, { @@ -857,6 +996,8 @@ export class MemoryCatalogRepository implements CatalogRepository { lastSyncedAt: now, createdAt: current?.createdAt ?? now, updatedAt: comparable === nextComparable ? (current?.updatedAt ?? now) : now, + origin: "physical", + status: "active", }); if (!current) created += 1; else if (comparable !== nextComparable) updated += 1; @@ -1073,6 +1214,12 @@ export class MemoryCatalogRepository implements CatalogRepository { this.relationships.delete(relationship.id); } } + for (const relationship of [...this.logicalRelationships.values()]) { + const pair = relationship.columns[0]; + if (pair.sourceColumnId === columnId || pair.targetColumnId === columnId) { + this.logicalRelationships.delete(relationship.id); + } + } } private markCatalogIncomplete(databaseIds: readonly string[]): void { diff --git a/backend/src/catalog/migrate.ts b/backend/src/catalog/migrate.ts index e6a0d179..8b6c7459 100644 --- a/backend/src/catalog/migrate.ts +++ b/backend/src/catalog/migrate.ts @@ -10,6 +10,7 @@ import * as catalogRuntimeSequencePrivilegesMigration from "./migrations/004_cat import * as descriptionGenerationRunsMigration from "./migrations/005_description_generation_runs.js"; import * as sensitiveDataFlagMigration from "./migrations/006_sensitive_data_flag.js"; import * as sensitiveDataSuggestionRunsMigration from "./migrations/007_sensitive_data_suggestion_runs.js"; +import * as catalogLogicalRelationshipsMigration from "./migrations/008_catalog_logical_relationships.js"; const connectionString = process.env.THT_CATALOG_MIGRATOR_DATABASE_URL; const host = process.env.THT_CATALOG_DB_HOST; @@ -42,6 +43,7 @@ const provider: MigrationProvider = { "005_description_generation_runs": descriptionGenerationRunsMigration, "006_sensitive_data_flag": sensitiveDataFlagMigration, "007_sensitive_data_suggestion_runs": sensitiveDataSuggestionRunsMigration, + "008_catalog_logical_relationships": catalogLogicalRelationshipsMigration, }; }, }; diff --git a/backend/src/catalog/migrations/008_catalog_logical_relationships.ts b/backend/src/catalog/migrations/008_catalog_logical_relationships.ts new file mode 100644 index 00000000..e375d59c --- /dev/null +++ b/backend/src/catalog/migrations/008_catalog_logical_relationships.ts @@ -0,0 +1,35 @@ +import { type Kysely, sql } from "kysely"; +import type { CatalogDatabase } from "../repository.js"; + +export async function up(db: Kysely): Promise { + await db.schema.createTable("catalog_logical_relationships") + .addColumn("id", "uuid", (column) => column.primaryKey()) + .addColumn("database_id", "uuid", (column) => column.notNull() + .references("workspace_databases.id").onDelete("cascade")) + .addColumn("source_column_id", "uuid", (column) => column.notNull() + .references("catalog_columns.id").onDelete("cascade")) + .addColumn("target_column_id", "uuid", (column) => column.notNull() + .references("catalog_columns.id").onDelete("cascade")) + .addColumn("generated", "boolean", (column) => column.notNull().defaultTo(false)) + .addColumn("deleted_at", "timestamptz") + .addColumn("created_at", "timestamptz", (column) => column.notNull().defaultTo(sql`now()`)) + .addColumn("updated_at", "timestamptz", (column) => column.notNull().defaultTo(sql`now()`)) + .addUniqueConstraint( + "catalog_logical_relationships_endpoint_key", + ["database_id", "source_column_id", "target_column_id"], + ) + .addCheckConstraint( + "catalog_logical_relationships_distinct_columns_check", + sql`source_column_id <> target_column_id`, + ) + .execute(); + + await db.schema.createIndex("catalog_logical_relationships_database_deleted_idx") + .on("catalog_logical_relationships") + .columns(["database_id", "deleted_at"]) + .execute(); +} + +export async function down(db: Kysely): Promise { + await db.schema.dropTable("catalog_logical_relationships").execute(); +} diff --git a/backend/src/catalog/repository.ts b/backend/src/catalog/repository.ts index d48063ca..79be4018 100644 --- a/backend/src/catalog/repository.ts +++ b/backend/src/catalog/repository.ts @@ -23,7 +23,10 @@ import { type CatalogDatabaseMetadataDeleteTarget, type CatalogMetadataDeleteCounts, type CatalogMetrics, - type CatalogRelationship, + type CatalogLogicalRelationship, + type CatalogLogicalRelationshipCandidate, + type CatalogLogicalRelationshipContext, + type CatalogPhysicalRelationship, type CatalogSchemaDiff, type CatalogSyncCounts, type CatalogSyncEvent, @@ -145,6 +148,17 @@ interface CatalogRelationshipColumnTable { targetColumnId: string; } +interface CatalogLogicalRelationshipTable { + id: string; + databaseId: string; + sourceColumnId: string; + targetColumnId: string; + generated: Generated; + deletedAt: Timestamp | null; + createdAt: Timestamp; + updatedAt: Timestamp; +} + interface DescriptionGenerationRunTable { id: string; databaseId: string; @@ -241,6 +255,7 @@ export interface CatalogDatabase { catalogColumns: CatalogColumnTable; catalogRelationships: CatalogRelationshipTable; catalogRelationshipColumns: CatalogRelationshipColumnTable; + catalogLogicalRelationships: CatalogLogicalRelationshipTable; descriptionGenerationRuns: DescriptionGenerationRunTable; descriptionGenerationEvents: DescriptionGenerationEventTable; sensitiveDataSuggestionRuns: SensitiveDataSuggestionRunTable; @@ -1022,7 +1037,7 @@ export class KyselyCatalogRepository implements CatalogRepository { return rows.map(serializeSensitiveDataSuggestionEvent); } - async listRelationships(databaseId: string): Promise { + async listRelationships(databaseId: string): Promise { const rows = await this.db.selectFrom("catalogRelationships as relationship") .innerJoin("catalogTables as sourceTable", "sourceTable.id", "relationship.sourceTableId") .innerJoin("catalogTables as targetTable", "targetTable.id", "relationship.targetTableId") @@ -1049,7 +1064,7 @@ export class KyselyCatalogRepository implements CatalogRepository { .where("pair.relationshipId", "in", rows.map((row) => row.id)) .orderBy("pair.relationshipId").orderBy("pair.position") .execute(); - const byRelationship = new Map(); + const byRelationship = new Map(); for (const pair of pairs) { const items = byRelationship.get(pair.relationshipId) ?? []; items.push(pair); @@ -1061,9 +1076,160 @@ export class KyselyCatalogRepository implements CatalogRepository { lastSyncedAt: row.lastSyncedAt === null ? null : new Date(row.lastSyncedAt).toISOString(), createdAt: new Date(row.createdAt).toISOString(), updatedAt: new Date(row.updatedAt).toISOString(), + origin: "physical" as const, + status: "active" as const, })); } + async listLogicalRelationships(databaseId: string): Promise { + const rows = await this.db.selectFrom("catalogLogicalRelationships as relationship") + .innerJoin("catalogColumns as sourceColumn", "sourceColumn.id", "relationship.sourceColumnId") + .innerJoin("catalogTables as sourceTable", "sourceTable.id", "sourceColumn.tableId") + .innerJoin("catalogColumns as targetColumn", "targetColumn.id", "relationship.targetColumnId") + .innerJoin("catalogTables as targetTable", "targetTable.id", "targetColumn.tableId") + .select([ + "relationship.id", "relationship.databaseId", "relationship.generated", + "relationship.deletedAt", "relationship.createdAt", "relationship.updatedAt", + "sourceTable.id as sourceTableId", "sourceTable.name as sourceTableName", + "sourceColumn.id as sourceColumnId", "sourceColumn.name as sourceColumnName", + "targetTable.id as targetTableId", "targetTable.name as targetTableName", + "targetColumn.id as targetColumnId", "targetColumn.name as targetColumnName", + ]) + .where("relationship.databaseId", "=", databaseId) + .orderBy("sourceTable.name") + .orderBy("sourceColumn.name") + .orderBy("targetTable.name") + .orderBy("targetColumn.name") + .execute(); + return rows.map((row) => ({ + id: row.id, + databaseId: row.databaseId, + constraintName: null, + sourceTableId: row.sourceTableId, + sourceTableName: row.sourceTableName, + targetTableId: row.targetTableId, + targetTableName: row.targetTableName, + updateRule: null, + deleteRule: null, + deferrable: false, + initiallyDeferred: false, + columns: [{ + position: 1, + sourceColumnId: row.sourceColumnId, + sourceColumnName: row.sourceColumnName, + targetColumnId: row.targetColumnId, + targetColumnName: row.targetColumnName, + }], + lastSyncedDatabaseVersion: null, + lastSyncedAt: null, + createdAt: new Date(row.createdAt).toISOString(), + updatedAt: new Date(row.updatedAt).toISOString(), + origin: row.generated ? "generated" : "manual", + status: row.deletedAt === null ? "active" : "excluded", + })); + } + + async getLogicalRelationshipContext( + databaseId: string, + ): Promise { + if (!(await selectOne(this.db, databaseId))) return undefined; + const columns = await this.db.selectFrom("catalogColumns as column") + .innerJoin("catalogTables as table", "table.id", "column.tableId") + .select([ + "column.id as columnId", "column.name as columnName", "column.dataType", + "column.primaryKeyPosition", "table.id as tableId", "table.name as tableName", + ]) + .where("table.databaseId", "=", databaseId) + .orderBy("table.name") + .orderBy("column.ordinalPosition") + .execute(); + const primaryKeyCounts = new Map(); + for (const column of columns) { + if (column.primaryKeyPosition !== null) { + primaryKeyCounts.set(column.tableId, (primaryKeyCounts.get(column.tableId) ?? 0) + 1); + } + } + const physicalPairs = await this.db.selectFrom("catalogRelationshipColumns as pair") + .innerJoin("catalogRelationships as relationship", "relationship.id", "pair.relationshipId") + .select(["pair.sourceColumnId", "pair.targetColumnId"]) + .where("relationship.databaseId", "=", databaseId) + .execute(); + return { + endpoints: columns.map((column) => ({ + ...column, + tablePrimaryKeyColumnCount: primaryKeyCounts.get(column.tableId) ?? 0, + })), + physicalPairs, + logicalRelationships: await this.listLogicalRelationships(databaseId), + }; + } + + async insertLogicalRelationship( + databaseId: string, + sourceColumnId: string, + targetColumnId: string, + generated: boolean, + ): Promise { + const id = randomUUID(); + const inserted = await this.db.insertInto("catalogLogicalRelationships").values({ + id, databaseId, sourceColumnId, targetColumnId, generated, + }).onConflict((conflict) => conflict + .columns(["databaseId", "sourceColumnId", "targetColumnId"]) + .doNothing()) + .returning("id") + .executeTakeFirst(); + if (!inserted) return undefined; + return (await this.listLogicalRelationships(databaseId)).find((item) => item.id === id); + } + + async insertGeneratedLogicalRelationships( + databaseId: string, + candidates: readonly CatalogLogicalRelationshipCandidate[], + ): Promise { + if (candidates.length === 0) return 0; + return await this.db.transaction().execute(async (trx) => { + let added = 0; + for (const candidate of candidates) { + const inserted = await trx.insertInto("catalogLogicalRelationships").values({ + id: randomUUID(), + databaseId, + sourceColumnId: candidate.sourceColumnId, + targetColumnId: candidate.targetColumnId, + generated: true, + }).onConflict((conflict) => conflict + .columns(["databaseId", "sourceColumnId", "targetColumnId"]) + .doNothing()) + .returning("id") + .executeTakeFirst(); + if (inserted) added += 1; + } + return added; + }); + } + + async setLogicalRelationshipStatus( + databaseId: string, + relationshipId: string, + status: CatalogLogicalRelationship["status"], + ): Promise { + const updated = await this.db.updateTable("catalogLogicalRelationships") + .set({ deletedAt: status === "excluded" ? sql`now()` : null, updatedAt: sql`now()` }) + .where("databaseId", "=", databaseId) + .where("id", "=", relationshipId) + .returning("id") + .executeTakeFirst(); + if (!updated) return undefined; + return (await this.listLogicalRelationships(databaseId)).find((item) => item.id === relationshipId); + } + + async deleteLogicalRelationship(databaseId: string, relationshipId: string): Promise { + const result = await this.db.deleteFrom("catalogLogicalRelationships") + .where("databaseId", "=", databaseId) + .where("id", "=", relationshipId) + .executeTakeFirst(); + return result.numDeletedRows > 0n; + } + async deleteDatabaseMetadata( databaseIds: readonly string[], target: CatalogDatabaseMetadataDeleteTarget, @@ -1621,7 +1787,13 @@ export class UnavailableCatalogRepository implements CatalogRepository { async updateSensitiveDataSuggestionRun(): Promise { return this.fail(); } async appendSensitiveDataSuggestionEvent(): Promise { return this.fail(); } async listSensitiveDataSuggestionEvents(): Promise { return this.fail(); } - async listRelationships(): Promise { return this.fail(); } + async listRelationships(): Promise { return this.fail(); } + async listLogicalRelationships(): Promise { return this.fail(); } + async getLogicalRelationshipContext(): Promise { return this.fail(); } + async insertLogicalRelationship(): Promise { return this.fail(); } + async insertGeneratedLogicalRelationships(): Promise { return this.fail(); } + async setLogicalRelationshipStatus(): Promise { return this.fail(); } + async deleteLogicalRelationship(): Promise { return this.fail(); } async deleteDatabaseMetadata(): Promise { return this.fail(); } async deleteTableMetadata(): Promise { return this.fail(); } async planSchemaSync(): Promise { return this.fail(); } diff --git a/backend/src/catalog/service.ts b/backend/src/catalog/service.ts index 723fadb7..17a4ae6f 100644 --- a/backend/src/catalog/service.ts +++ b/backend/src/catalog/service.ts @@ -2,6 +2,7 @@ import { buildInstallationContract } from "../workspaces/contracts.js"; import { resolveBinding } from "../workspaces/bindings.js"; import type { WorkspaceRegistry } from "../workspaces/registry.js"; import type { WorkspaceDescriptor } from "../workspaces/schema.js"; +import { discoverWorkspaceSecretRequirements } from "../workspaces/secret-requirements.js"; import type { WorkspaceSecretStore } from "../workspaces/secret-store.js"; import { createConcreteDiagnosticAdapters } from "../workspaces/diagnostics.js"; import { CatalogOperationCoordinator } from "./operation-coordinator.js"; @@ -25,6 +26,21 @@ export interface CatalogListItem extends Omit { workspaceName: string; workspaceDescription?: string; workspaceAvailable: boolean; + workspaceRevision: { commit: string; blob: string } | null; + workspaceEvidence: { + sourceType: "filesystem" | "http" | "s3" | null; + state: + | "not_declared" + | "materialized_current_revision" + | "configuration_required" + | "configured_unverified" + | "workspace_unavailable"; + }; + runtimeBinding: { + transport: DatabaseBinding["transport"]; + configurationState: "ready" | "configuration_required"; + sessionTransportSupported: boolean; + } | null; configured: boolean; secrets: Record; } @@ -68,6 +84,43 @@ function secretState(store: WorkspaceSecretStore, workspaceId: string): Record; } +function workspaceRuntimeState( + store: WorkspaceSecretStore, + workspace: WorkspaceDescriptor, + secretRoots: readonly string[], +): NonNullable { + const requirements = discoverWorkspaceSecretRequirements(workspace, process.env); + const secretVariables = new Set(requirements.map(({ variable }) => variable)); + const effective = resolveBinding(workspace, "DWH", process.env, secretRoots); + const configurationRequired = requirements.some(({ id, required }) => ( + required && !store.has(workspace.workspace.id, id) + )) || effective.missing.some((variable) => !secretVariables.has(variable)); + return { + transport: effective.transport, + configurationState: configurationRequired ? "configuration_required" : "ready", + sessionTransportSupported: effective.transport !== "ssh_tunnel", + }; +} + +function workspaceEvidenceState( + store: WorkspaceSecretStore, + workspace: WorkspaceDescriptor, +): CatalogListItem["workspaceEvidence"] { + const source = workspace.evidence?.source; + if (!source) return { sourceType: null, state: "not_declared" }; + if (source.type === "filesystem") { + return { sourceType: source.type, state: "materialized_current_revision" }; + } + const configurationRequired = discoverWorkspaceSecretRequirements(workspace, process.env) + .some(({ connector, id, required }) => ( + connector === "evidence" && required && !store.has(workspace.workspace.id, id) + )); + return { + sourceType: source.type, + state: configurationRequired ? "configuration_required" : "configured_unverified", + }; +} + export class CatalogService { private readonly adapters = createConcreteDiagnosticAdapters(); @@ -91,7 +144,8 @@ export class CatalogService { const byWorkspace = new Map(configured.map((database) => [database.workspaceId, database])); const active = await Promise.all(workspaces.map(async (entry) => { const database = byWorkspace.get(entry.id); - const { workspace } = await this.registry.read(entry.id); + const { workspace } = await this.registry.readPinned(entry.id, entry.revision.commit); + const runtimeDatabaseBinding = yamlBinding(workspace, this.secretRoots); const base = database ?? { workspaceId: entry.id, engine: "postgres" as const, @@ -100,7 +154,7 @@ export class CatalogService { version: 0, createdAt: "", updatedAt: "", - binding: yamlBinding(workspace, this.secretRoots), + binding: runtimeDatabaseBinding, connectionStatus: "untested" as const, }; return { @@ -108,6 +162,12 @@ export class CatalogService { workspaceName: entry.name, workspaceDescription: entry.description, workspaceAvailable: true, + workspaceRevision: { + commit: entry.revision.commit, + blob: entry.revision.blob, + }, + workspaceEvidence: workspaceEvidenceState(this.secretStore, workspace), + runtimeBinding: workspaceRuntimeState(this.secretStore, workspace, this.secretRoots), configured: database !== undefined, secrets: secretState(this.secretStore, entry.id), }; @@ -120,6 +180,9 @@ export class CatalogService { workspaceName: database.workspaceId, workspaceDescription: "Workspace is no longer present in the repository catalog.", workspaceAvailable: false, + workspaceRevision: null, + workspaceEvidence: { sourceType: null, state: "workspace_unavailable" }, + runtimeBinding: null, configured: true, secrets: secretState(this.secretStore, database.workspaceId), })); diff --git a/backend/src/catalog/types.ts b/backend/src/catalog/types.ts index 809af9f6..86fdf9d2 100644 --- a/backend/src/catalog/types.ts +++ b/backend/src/catalog/types.ts @@ -128,7 +128,7 @@ export interface CatalogRelationshipColumn { targetColumnName: string; } -export interface CatalogRelationship { +export interface CatalogPhysicalRelationship { id: string; databaseId: string; constraintName: string; @@ -145,6 +145,52 @@ export interface CatalogRelationship { lastSyncedAt: string | null; createdAt: string; updatedAt: string; + origin: "physical"; + status: "active"; +} + +export interface CatalogLogicalRelationship { + id: string; + databaseId: string; + constraintName: null; + sourceTableId: string; + sourceTableName: string; + targetTableId: string; + targetTableName: string; + updateRule: null; + deleteRule: null; + deferrable: false; + initiallyDeferred: false; + columns: [CatalogRelationshipColumn]; + lastSyncedDatabaseVersion: null; + lastSyncedAt: null; + createdAt: string; + updatedAt: string; + origin: "generated" | "manual"; + status: "active" | "excluded"; +} + +export type CatalogRelationship = CatalogPhysicalRelationship | CatalogLogicalRelationship; + +export interface CatalogLogicalRelationshipEndpoint { + columnId: string; + columnName: string; + tableId: string; + tableName: string; + dataType: string; + primaryKeyPosition: number | null; + tablePrimaryKeyColumnCount: number; +} + +export interface CatalogLogicalRelationshipContext { + endpoints: CatalogLogicalRelationshipEndpoint[]; + physicalPairs: Array<{ sourceColumnId: string; targetColumnId: string }>; + logicalRelationships: CatalogLogicalRelationship[]; +} + +export interface CatalogLogicalRelationshipCandidate { + sourceColumnId: string; + targetColumnId: string; } export type CatalogDatabaseMetadataDeleteTarget = "tables" | "relationships"; @@ -456,7 +502,25 @@ export interface CatalogRepository { runId: string, afterSequence?: number, ): Promise; - listRelationships(databaseId: string): Promise; + listRelationships(databaseId: string): Promise; + listLogicalRelationships(databaseId: string): Promise; + getLogicalRelationshipContext(databaseId: string): Promise; + insertLogicalRelationship( + databaseId: string, + sourceColumnId: string, + targetColumnId: string, + generated: boolean, + ): Promise; + insertGeneratedLogicalRelationships( + databaseId: string, + candidates: readonly CatalogLogicalRelationshipCandidate[], + ): Promise; + setLogicalRelationshipStatus( + databaseId: string, + relationshipId: string, + status: CatalogLogicalRelationship["status"], + ): Promise; + deleteLogicalRelationship(databaseId: string, relationshipId: string): Promise; deleteDatabaseMetadata( databaseIds: readonly string[], target: CatalogDatabaseMetadataDeleteTarget, diff --git a/backend/src/routes/catalog-logical-relationships.ts b/backend/src/routes/catalog-logical-relationships.ts new file mode 100644 index 00000000..167a2289 --- /dev/null +++ b/backend/src/routes/catalog-logical-relationships.ts @@ -0,0 +1,154 @@ +import type { FastifyInstance, FastifyReply, FastifyRequest } from "fastify"; +import { z } from "zod"; +import { isPrincipalContext, requirePermission } from "../auth/authorization.js"; +import { + CatalogLogicalRelationshipService, + LogicalRelationshipColumnNotFoundError, + LogicalRelationshipDatabaseNotFoundError, + LogicalRelationshipDuplicateError, + LogicalRelationshipNotFoundError, + LogicalRelationshipReadOnlyError, + LogicalRelationshipSchemaStaleError, + LogicalRelationshipTargetNotUniqueError, + LogicalRelationshipTypeIncompatibleError, +} from "../catalog/logical-relationship-service.js"; +import type { CatalogOperationCoordinator } from "../catalog/operation-coordinator.js"; +import { CatalogOperationInProgressError, CatalogUnavailableError } from "../catalog/types.js"; + +const idSchema = z.uuid(); +const createSchema = z.object({ + sourceColumnId: idSchema, + targetColumnId: idSchema, +}).strict(); +const statusSchema = z.object({ status: z.enum(["active", "excluded"]) }).strict(); +const emptySchema = z.object({}).strict(); + +function manage(request: FastifyRequest, reply: FastifyReply) { + return isPrincipalContext(requirePermission(request, reply, "database.manage")); +} + +function safeError(reply: FastifyReply, error: unknown) { + if (error instanceof CatalogUnavailableError) { + return reply.code(503).send({ code: "catalog_unavailable", message: "Database catalog is unavailable." }); + } + if (error instanceof CatalogOperationInProgressError) { + return reply.code(409).send({ + code: "database_operation_in_progress", + message: "A database operation is already in progress.", + }); + } + if (error instanceof LogicalRelationshipDatabaseNotFoundError) { + return reply.code(404).send({ code: "database_not_found", message: "Database configuration was not found." }); + } + if (error instanceof LogicalRelationshipColumnNotFoundError) { + return reply.code(404).send({ + code: "column_not_found", + message: "Catalog column was not found.", + field: error.field, + }); + } + if (error instanceof LogicalRelationshipNotFoundError) { + return reply.code(404).send({ code: "relationship_not_found", message: "Relationship was not found." }); + } + if (error instanceof LogicalRelationshipDuplicateError) { + return reply.code(409).send({ code: "relationship_duplicate", message: "Relationship already exists." }); + } + if (error instanceof LogicalRelationshipReadOnlyError) { + return reply.code(409).send({ code: "relationship_read_only", message: "Physical relationships are read-only." }); + } + if (error instanceof LogicalRelationshipSchemaStaleError) { + return reply.code(409).send({ + code: "relationship_schema_stale", + message: "Synchronize the current database schema before managing logical relationships.", + }); + } + if (error instanceof LogicalRelationshipTargetNotUniqueError) { + return reply.code(422).send({ + code: "relationship_target_not_unique", + message: "Target column must be the only primary-key column of its table.", + field: "targetColumnId", + }); + } + if (error instanceof LogicalRelationshipTypeIncompatibleError) { + return reply.code(422).send({ + code: "relationship_type_incompatible", + message: "Source and target column types are incompatible.", + field: "targetColumnId", + }); + } + if (error instanceof z.ZodError) { + return reply.code(400).send({ + code: "relationship_request_invalid", + message: "Relationship request is invalid.", + }); + } + return reply.code(500).send({ + code: "relationship_operation_failed", + message: "Relationship operation failed.", + }); +} + +export function catalogLogicalRelationshipRoutes( + app: FastifyInstance, + deps: { + service: CatalogLogicalRelationshipService; + operations: CatalogOperationCoordinator; + }, +): void { + app.get("/catalog/databases/:databaseId/relationships", async (request, reply) => { + if (!manage(request, reply)) return reply; + try { + const databaseId = idSchema.parse((request.params as { databaseId?: unknown }).databaseId); + return await deps.service.list(databaseId); + } catch (error) { return safeError(reply, error); } + }); + + app.post("/catalog/databases/:databaseId/relationships", async (request, reply) => { + if (!manage(request, reply)) return reply; + try { + const databaseId = idSchema.parse((request.params as { databaseId?: unknown }).databaseId); + const input = createSchema.parse(request.body); + const relationship = await deps.operations.run(databaseId, async () => ( + await deps.service.addManual(databaseId, input.sourceColumnId, input.targetColumnId) + )); + return reply.code(201).send(relationship); + } catch (error) { return safeError(reply, error); } + }); + + app.post("/catalog/databases/:databaseId/relationships/rebuild-generated", async (request, reply) => { + if (!manage(request, reply)) return reply; + try { + const databaseId = idSchema.parse((request.params as { databaseId?: unknown }).databaseId); + emptySchema.parse(request.body ?? {}); + return await deps.operations.run(databaseId, async () => ( + await deps.service.rebuildGenerated(databaseId) + )); + } catch (error) { return safeError(reply, error); } + }); + + app.patch("/catalog/databases/:databaseId/relationships/:relationshipId", async (request, reply) => { + if (!manage(request, reply)) return reply; + try { + const params = request.params as { databaseId?: unknown; relationshipId?: unknown }; + const databaseId = idSchema.parse(params.databaseId); + const relationshipId = idSchema.parse(params.relationshipId); + const input = statusSchema.parse(request.body); + return await deps.operations.run(databaseId, async () => ( + await deps.service.setStatus(databaseId, relationshipId, input.status) + )); + } catch (error) { return safeError(reply, error); } + }); + + app.delete("/catalog/databases/:databaseId/relationships/:relationshipId", async (request, reply) => { + if (!manage(request, reply)) return reply; + try { + const params = request.params as { databaseId?: unknown; relationshipId?: unknown }; + const databaseId = idSchema.parse(params.databaseId); + const relationshipId = idSchema.parse(params.relationshipId); + await deps.operations.run(databaseId, async () => { + await deps.service.deletePermanently(databaseId, relationshipId); + }); + return reply.code(204).send(); + } catch (error) { return safeError(reply, error); } + }); +} diff --git a/backend/src/routes/catalog-schema.ts b/backend/src/routes/catalog-schema.ts index 3c17e8a5..d5182b51 100644 --- a/backend/src/routes/catalog-schema.ts +++ b/backend/src/routes/catalog-schema.ts @@ -131,17 +131,6 @@ export function catalogSchemaRoutes( } catch (error) { return safeError(reply, error); } }); - app.get("/catalog/databases/:databaseId/relationships", async (request, reply) => { - if (!manage(request, reply)) return reply; - try { - const databaseId = idSchema.parse((request.params as { databaseId?: unknown }).databaseId); - if (!(await deps.repository.get(databaseId))) { - return reply.code(404).send({ code: "database_not_found", message: "Database configuration was not found." }); - } - return await deps.repository.listRelationships(databaseId); - } catch (error) { return safeError(reply, error); } - }); - app.post("/catalog/databases/metadata-cleanup", async (request, reply) => { if (!manage(request, reply)) return reply; try { diff --git a/backend/src/routes/sessions.ts b/backend/src/routes/sessions.ts index c7d5ed84..6ca529ca 100644 --- a/backend/src/routes/sessions.ts +++ b/backend/src/routes/sessions.ts @@ -11,6 +11,7 @@ import type { WorkspaceRegistry } from "../workspaces/registry.js"; import { validateOperationalWorkspace, type WorkspaceDescriptor } from "../workspaces/schema.js"; import type { MaintenanceBarrier } from "../runtime/maintenance-gate.js"; import { hasPermission, isPrincipalContext, requirePermission } from "../auth/authorization.js"; +import type { EffectiveRelationshipSnapshotProvider } from "../catalog/effective-relationship-snapshot.js"; const BOOTSTRAP_FAILURE_MESSAGE = "Session startup failed. Check configuration and connectivity, then Resume the session."; @@ -40,6 +41,8 @@ export function sessionRoutes( /** Fail-closed installation/runtime transport capability check. */ workspaceRuntimeSupport: (workspace: WorkspaceDescriptor) => boolean; maintenanceBarrier: MaintenanceBarrier; + /** Optional only for narrow route-test stubs and installations without a Catalog database. */ + effectiveRelationships?: EffectiveRelationshipSnapshotProvider; }, ) { const lifecycleTails = new Map>(); @@ -84,11 +87,21 @@ export function sessionRoutes( isAdmin: hasPermission(principal, permission), }); - const optionsWithRuntimeConfig = (runner: any, workspaceConfigPath: string | undefined, options: any) => ( - workspaceConfigPath && typeof runner.acquireWorkspaceRuntime === "function" - ? { ...options, runtimeConfig: runner.acquireWorkspaceRuntime(workspaceConfigPath) } - : options - ); + const optionsWithRuntimeConfig = async ( + runner: any, + workspaceConfigPath: string | undefined, + workspaceId: string | undefined, + options: any, + ) => { + if (!workspaceConfigPath || typeof runner.acquireWorkspaceRuntime !== "function") return options; + const effectiveRelationships = workspaceId && d.effectiveRelationships + ? await d.effectiveRelationships.render(workspaceId) + : undefined; + return { + ...options, + runtimeConfig: runner.acquireWorkspaceRuntime(workspaceConfigPath, effectiveRelationships), + }; + }; const maintenanceReply = (reply: any) => reply.code(503).send({ code: "maintenance", @@ -448,7 +461,12 @@ export function sessionRoutes( let runtimeOptions = options; let rt: ReturnType | undefined; try { - runtimeOptions = optionsWithRuntimeConfig(runner, workspaceConfigPath, options); + runtimeOptions = await optionsWithRuntimeConfig( + runner, + workspaceConfigPath, + workspaceId, + options, + ); rt = d.mgr.createFor(id, runtimeOptions); bindRuntime(id, rt, runner, workspaceConfigPath); } catch (error) { @@ -465,7 +483,11 @@ export function sessionRoutes( info(id, "Session created"); bootstrap( id, rt, runner, workspaceConfigPath, d.mgr.configure(rt, runtimeOptions), - runner.searchPack(b.question, id, workspaceConfigPath), + runner.searchPack( + b.question, + id, + (runtimeOptions as any).runtimeConfig?.path ?? workspaceConfigPath, + ), () => d.mgr.start(id, rt, runtimeOptions), ); return { id }; @@ -639,7 +661,12 @@ export function sessionRoutes( if (boundRuntimes.get(id) === current) boundRuntimes.delete(id); d.mgr.teardownIfCurrent(id, current); } - runtimeOptions = optionsWithRuntimeConfig(runner, workspaceConfigPath, options); + runtimeOptions = await optionsWithRuntimeConfig( + runner, + workspaceConfigPath, + saved.workspace_id, + options, + ); rt = d.mgr.createFor(id, runtimeOptions); bindRuntime(id, rt, runner, workspaceConfigPath); } catch { diff --git a/backend/src/tht/tht-runner.ts b/backend/src/tht/tht-runner.ts index 4be2e0b0..2074bdfd 100644 --- a/backend/src/tht/tht-runner.ts +++ b/backend/src/tht/tht-runner.ts @@ -213,23 +213,37 @@ export class ThtRunner { } /** Render one immutable canonical registry revision into a backend-owned harness config. */ - acquireWorkspaceRuntime(workspaceConfigPath: string): RuntimeConfigLease { - const rendered = renderWorkspaceRuntimeFromSnapshotPath({ - snapshotPath: workspaceConfigPath, - harnessDir: this.cfg.harnessDir, - configPath: this.cfg.configPath, - dataRoot: this.cfg.dataRoot ?? (() => { - throw new Error("registry workspace runtime requires an absolute data root"); - })(), - secretRoots: this.cfg.secretRoots ?? [], - semanticRuntime: this.cfg.semanticRuntime ?? DEFAULT_SEMANTIC_RUNTIME, - workspaceSecretStore: this.cfg.workspaceSecretStore, - }); + acquireWorkspaceRuntime( + workspaceConfigPath: string, + effectiveRelationships?: string, + ): RuntimeConfigLease { + const effectiveRelationshipsPath = effectiveRelationships === undefined + ? undefined + : this.createRuntimeSnapshot(effectiveRelationships); + let rendered: ReturnType; + try { + rendered = renderWorkspaceRuntimeFromSnapshotPath({ + snapshotPath: workspaceConfigPath, + harnessDir: this.cfg.harnessDir, + configPath: this.cfg.configPath, + dataRoot: this.cfg.dataRoot ?? (() => { + throw new Error("registry workspace runtime requires an absolute data root"); + })(), + secretRoots: this.cfg.secretRoots ?? [], + semanticRuntime: this.cfg.semanticRuntime ?? DEFAULT_SEMANTIC_RUNTIME, + workspaceSecretStore: this.cfg.workspaceSecretStore, + effectiveRelationshipsPath, + }); + } catch (error) { + if (effectiveRelationshipsPath) this.cleanupRuntimeSnapshot(effectiveRelationshipsPath); + throw error; + } let path: string; try { path = this.createRuntimeSnapshot(rendered.renderedConfig); } catch (error) { rendered.releaseSecrets(); + if (effectiveRelationshipsPath) this.cleanupRuntimeSnapshot(effectiveRelationshipsPath); throw error; } let released = false; @@ -241,6 +255,7 @@ export class ThtRunner { if (released) return; released = true; this.cleanupRuntimeSnapshot(path); + if (effectiveRelationshipsPath) this.cleanupRuntimeSnapshot(effectiveRelationshipsPath); rendered.releaseSecrets(); }, }; diff --git a/backend/src/workspaces/runtime-config-lease.ts b/backend/src/workspaces/runtime-config-lease.ts index 912fb872..b1f8a535 100644 --- a/backend/src/workspaces/runtime-config-lease.ts +++ b/backend/src/workspaces/runtime-config-lease.ts @@ -243,7 +243,12 @@ function readSnapshotWorkspace(snapshotPath: string): { } } -function runtimePaths(dataRoot: string, workspaceId: string, workspaceRevision?: string): RuntimePaths { +function runtimePaths( + dataRoot: string, + workspaceId: string, + workspaceRevision?: string, + effectiveRelationshipsPath?: string, +): RuntimePaths { if (!isAbsolute(dataRoot)) throw new Error("registry workspace runtime requires an absolute data root"); const root = join(dataRoot, "sessions", workspaceId); return { @@ -254,6 +259,9 @@ function runtimePaths(dataRoot: string, workspaceId: string, workspaceRevision?: ...(workspaceRevision === undefined ? {} : { annotations_root: join(dataRoot, "sessions", workspaceId, "revisions", workspaceRevision, "artifacts") }), + ...(effectiveRelationshipsPath === undefined + ? {} + : { effective_relationships: effectiveRelationshipsPath }), }; } @@ -301,6 +309,7 @@ function renderWorkspaceRuntimeFromWorkspace(options: { dataRoot: string; secretRoots: readonly string[]; semanticRuntime: SemanticRuntimeConfig; + effectiveRelationshipsPath?: string; workspaceSecretStore?: WorkspaceSecretStore; }): RenderedWorkspaceRuntime { const secretLease = options.workspaceSecretStore === undefined @@ -325,7 +334,12 @@ function renderWorkspaceRuntimeFromWorkspace(options: { workspaceId: options.workspaceId, workspaceRevision: options.workspaceRevision, revisionContentRoot: options.revisionContentRoot, - runtimePaths: runtimePaths(options.dataRoot, options.workspaceId, options.workspaceRevision), + runtimePaths: runtimePaths( + options.dataRoot, + options.workspaceId, + options.workspaceRevision, + options.effectiveRelationshipsPath, + ), installationOverlay: overlay, bindings, bindingDigest: stableBindingDigest(bindings), @@ -334,7 +348,12 @@ function renderWorkspaceRuntimeFromWorkspace(options: { renderedConfig: renderRuntimeConfig( options.workspace, bindings, - runtimePaths(options.dataRoot, options.workspaceId, options.workspaceRevision), + runtimePaths( + options.dataRoot, + options.workspaceId, + options.workspaceRevision, + options.effectiveRelationshipsPath, + ), context, overlay, options.semanticRuntime, @@ -353,6 +372,7 @@ export function renderWorkspaceRuntimeFromSnapshotPath(options: { dataRoot: string; secretRoots: readonly string[]; semanticRuntime: SemanticRuntimeConfig; + effectiveRelationshipsPath?: string; workspaceSecretStore?: WorkspaceSecretStore; }): RenderedWorkspaceRuntime { const snapshot = readSnapshotWorkspace(options.snapshotPath); @@ -366,6 +386,7 @@ export function renderWorkspaceRuntimeFromSnapshotPath(options: { dataRoot: options.dataRoot, secretRoots: options.secretRoots, semanticRuntime: options.semanticRuntime, + effectiveRelationshipsPath: options.effectiveRelationshipsPath, workspaceSecretStore: options.workspaceSecretStore, }); } diff --git a/backend/src/workspaces/runtime-renderer.ts b/backend/src/workspaces/runtime-renderer.ts index 4af55d8b..e1b3c5b0 100644 --- a/backend/src/workspaces/runtime-renderer.ts +++ b/backend/src/workspaces/runtime-renderer.ts @@ -12,6 +12,8 @@ export interface RuntimePaths { memory: string; /** Revision-qualified root for curated FK annotations (P5); optional for legacy callers. */ annotations_root?: string; + /** Immutable Catalog projection used as the exclusive runtime relationship source. */ + effective_relationships?: string; } export interface RuntimeIdentity { diff --git a/backend/test/catalog-databases-routes.test.ts b/backend/test/catalog-databases-routes.test.ts index fca44702..f40f0c34 100644 --- a/backend/test/catalog-databases-routes.test.ts +++ b/backend/test/catalog-databases-routes.test.ts @@ -13,7 +13,10 @@ import type { WorkspaceRegistry, WorkspaceRevision } from "../src/workspaces/reg import type { WorkspaceDescriptor } from "../src/workspaces/schema.js"; const roots: string[] = []; -afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); }); +afterEach(() => { + vi.unstubAllEnvs(); + for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); +}); const workspace: WorkspaceDescriptor = { workspace: { schema_version: 3, id: "psd-clinical", name: "Policlinico San Donato", language: "it" }, @@ -36,6 +39,7 @@ function setup( catalogOperationCoordinator?: CatalogOperationCoordinator; catalogPostgresAccess?: CatalogPostgresAccess; } = {}, + workspaceDescriptor: WorkspaceDescriptor = workspace, ) { const secretRoot = mkdtempSync(join(tmpdir(), "catalog-secret-")); const runtimeRoot = mkdtempSync(join(tmpdir(), "catalog-secret-runtime-")); @@ -45,7 +49,8 @@ function setup( const registry = { list: vi.fn(async () => [revision]), listCatalog: vi.fn(async () => [{ id: "psd-clinical", name: "Policlinico San Donato", configurationState: "ready", revision }]), - read: vi.fn(async () => ({ workspace, revision })), + read: vi.fn(async () => ({ workspace: workspaceDescriptor, revision })), + readPinned: vi.fn(async () => ({ workspace: workspaceDescriptor, workspaceConfigPath: revision.snapshotPath })), } as unknown as WorkspaceRegistry; const app = buildApp(loadConfig({ THT_HARNESS_DIR: "/missing", @@ -96,10 +101,31 @@ const fleetSnapshot: ObservedSchemaSnapshot = { }; test("lists every YAML workspace and creates its one database configuration", async () => { - const { app } = setup(); + const { app, secretStore } = setup(); const initial = await app.inject({ method: "GET", url: "/catalog/databases" }); expect(initial.statusCode).toBe(200); - expect(initial.json()).toMatchObject([{ workspaceId: "psd-clinical", configured: false, databaseName: "warehouse" }]); + expect(initial.json()).toMatchObject([{ + workspaceId: "psd-clinical", + configured: false, + databaseName: "warehouse", + workspaceRevision: { commit: revision.commit, blob: revision.blob }, + workspaceEvidence: { sourceType: null, state: "not_declared" }, + runtimeBinding: { + transport: "postgres_direct", + configurationState: "configuration_required", + sessionTransportSupported: true, + }, + }]); + + vi.stubEnv("THT_WS_PSD_CLINICAL_DWH_HOST", "runtime-db.internal"); + vi.stubEnv("THT_WS_PSD_CLINICAL_DWH_PORT", "5432"); + vi.stubEnv("THT_WS_PSD_CLINICAL_DWH_USER", "runtime-reader"); + vi.stubEnv("THT_WS_PSD_CLINICAL_DWH_TRANSPORT", "postgres_direct"); + secretStore.putMany("psd-clinical", { "dwh.password": "runtime-password" }); + const runtimeReady = await app.inject({ method: "GET", url: "/catalog/databases" }); + expect(runtimeReady.json()).toMatchObject([{ + runtimeBinding: { configurationState: "ready", sessionTransportSupported: true }, + }]); const created = await app.inject({ method: "POST", url: "/catalog/databases", payload: direct }); expect(created.statusCode).toBe(201); @@ -110,6 +136,39 @@ test("lists every YAML workspace and creates its one database configuration", as expect(listed.json()).toMatchObject([{ configured: true, binding: { transport: "postgres_direct", host: "db.internal" } }]); }); +test("projects remote Evidence credential state without conflating catalog secrets", async () => { + const evidenceWorkspace: WorkspaceDescriptor = { + ...workspace, + evidence: { + schema_version: 2, + source: { + type: "http", + uris: ["https://evidence.example.test/guide.md"], + authentication: "signed_urls_file", + connect_timeout_ms: 5_000, + read_timeout_ms: 30_000, + max_bytes: 10 * 1024 * 1024, + max_redirects: 5, + allow_private_hosts: false, + max_cache_bytes: 64 * 1024 * 1024, + }, + policy: { max_chunk_chars: 4_000, retain_published_generations: 3 }, + }, + }; + const { app, secretStore } = setup({}, {}, evidenceWorkspace); + + const missing = await app.inject({ method: "GET", url: "/catalog/databases" }); + expect(missing.json()).toMatchObject([{ + workspaceEvidence: { sourceType: "http", state: "configuration_required" }, + }]); + + secretStore.putMany("psd-clinical", { "evidence.signed_urls": "https://signed.example.test/evidence" }); + const configured = await app.inject({ method: "GET", url: "/catalog/databases" }); + expect(configured.json()).toMatchObject([{ + workspaceEvidence: { sourceType: "http", state: "configured_unverified" }, + }]); +}); + test("lists orphaned records and takes the REST diagnostic path from workspace YAML", async () => { const { app, repository } = setup(); await repository.create({ diff --git a/backend/test/catalog-logical-relationship-service.test.ts b/backend/test/catalog-logical-relationship-service.test.ts new file mode 100644 index 00000000..a108a6cc --- /dev/null +++ b/backend/test/catalog-logical-relationship-service.test.ts @@ -0,0 +1,207 @@ +import { expect, test } from "vitest"; +import { + CatalogLogicalRelationshipService, + LogicalRelationshipDuplicateError, + LogicalRelationshipSchemaStaleError, + LogicalRelationshipTargetNotUniqueError, + LogicalRelationshipTypeIncompatibleError, +} from "../src/catalog/logical-relationship-service.js"; +import { MemoryCatalogRepository } from "../src/catalog/memory-repository.js"; +import type { ObservedSchemaSnapshot } from "../src/catalog/types.js"; + +const column = ( + tableName: string, + name: string, + ordinalPosition: number, + dataType: string, + primaryKeyPosition: number | null, +) => ({ + tableName, name, ordinalPosition, dataType, primaryKeyPosition, + isNullable: false, defaultExpression: null, sourceComment: null, +}); + +function schema( + tableNames: string[], + columns: ObservedSchemaSnapshot["columns"], + relationships: ObservedSchemaSnapshot["relationships"] = [], +): ObservedSchemaSnapshot { + return { + schemaVersion: 1, + capabilities: { tables: "available", columns: "available", relationships: "available" }, + tables: tableNames.map((name) => ({ name, sourceComment: null })), + columns, + relationships, + }; +} + +async function setup(snapshot: ObservedSchemaSnapshot) { + const repository = new MemoryCatalogRepository(); + const database = await repository.create({ + workspaceId: "relationships", engine: "postgres", databaseName: "warehouse", schema: "public", + binding: { transport: "postgres_direct", host: "db.internal", port: 5432, username: "reader" }, + }); + await repository.applySchemaSync(database.id, database.version, "all", [], snapshot); + const service = new CatalogLogicalRelationshipService(repository); + const context = (await repository.getLogicalRelationshipContext(database.id))!; + const endpoint = (tableName: string, columnName: string) => context.endpoints.find((item) => ( + item.tableName === tableName && item.columnName === columnName + ))!; + return { repository, database, service, endpoint }; +} + +test("keeps excluded relationships across rebuild and recreates hard-deleted relationships", async () => { + const { database, service, endpoint } = await setup(schema( + ["users", "orders"], + [column("users", "id", 1, "bigint", 1), column("orders", "id", 1, "bigint", 1), + column("orders", "user_id", 2, "bigint", null)], + )); + const source = endpoint("orders", "user_id"); + const target = endpoint("users", "id"); + const manual = await service.addManual(database.id, source.columnId, target.columnId); + expect(manual).toMatchObject({ origin: "manual", status: "active" }); + + expect(await service.setStatus(database.id, manual.id, "excluded")) + .toMatchObject({ status: "excluded" }); + await expect(service.rebuildGenerated(database.id)).resolves.toEqual({ + added: 0, alreadyPresent: 0, excluded: 1, ambiguous: 0, + }); + expect((await service.list(database.id)).filter((item) => item.origin !== "physical")) + .toMatchObject([{ id: manual.id, origin: "manual", status: "excluded" }]); + + await service.deletePermanently(database.id, manual.id); + await expect(service.rebuildGenerated(database.id)).resolves.toEqual({ + added: 1, alreadyPresent: 0, excluded: 0, ambiguous: 0, + }); + expect((await service.list(database.id)).filter((item) => item.origin !== "physical")) + .toMatchObject([{ origin: "generated", status: "active" }]); +}); + +test("normalizes snake, kebab, and camel names without fuzzy matching", async () => { + const { database, service } = await setup(schema( + ["users", "events"], + [column("users", "id", 1, "bigint", 1), column("events", "id", 1, "bigint", 1), + column("events", "user_id", 2, "bigint", null), + column("events", "user-id", 3, "bigint", null), + column("events", "userId", 4, "bigint", null), + column("events", "userid", 5, "bigint", null), + column("events", "unrelated", 6, "bigint", null)], + )); + await expect(service.rebuildGenerated(database.id)).resolves.toEqual({ + added: 4, alreadyPresent: 0, excluded: 0, ambiguous: 0, + }); +}); + +test("skips generic bare primary-key names while retaining table-qualified matches", async () => { + const { database, service } = await setup(schema( + ["users", "accounts", "events"], + [column("users", "id", 1, "bigint", 1), column("accounts", "id", 1, "bigint", 1), + column("events", "event_key", 1, "bigint", 1), column("events", "id", 2, "bigint", null), + column("events", "user_id", 3, "bigint", null)], + )); + await expect(service.rebuildGenerated(database.id)).resolves.toEqual({ + added: 1, alreadyPresent: 0, excluded: 0, ambiguous: 0, + }); +}); + +test("infers foreign keys from composite-primary-key sources", async () => { + const { database, service } = await setup(schema( + ["users", "groups", "memberships"], + [column("users", "id", 1, "bigint", 1), column("groups", "id", 1, "bigint", 1), + column("memberships", "user_id", 1, "bigint", 1), + column("memberships", "group_id", 2, "bigint", 2)], + )); + await expect(service.rebuildGenerated(database.id)).resolves.toEqual({ + added: 2, alreadyPresent: 0, excluded: 0, ambiguous: 0, + }); +}); + +test("maps time-key columns to the single primary key of dim_time", async () => { + const { database, service } = await setup(schema( + ["dim_time", "admissions"], + [column("dim_time", "day_key", 1, "integer", 1), + column("admissions", "id", 1, "bigint", 1), + column("admissions", "admission_time_key", 2, "integer", null), + column("admissions", "discharge_time_key", 3, "integer", null)], + )); + await expect(service.rebuildGenerated(database.id)).resolves.toEqual({ + added: 2, alreadyPresent: 0, excluded: 0, ambiguous: 0, + }); +}); + +test("skips ambiguous targets and physical foreign-key pairs", async () => { + const ambiguous = await setup(schema( + ["user", "users", "events"], + [column("user", "id", 1, "bigint", 1), column("users", "id", 1, "bigint", 1), + column("events", "id", 1, "bigint", 1), column("events", "user_id", 2, "bigint", null)], + )); + await expect(ambiguous.service.rebuildGenerated(ambiguous.database.id)).resolves.toEqual({ + added: 0, alreadyPresent: 0, excluded: 0, ambiguous: 1, + }); + + const physical = await setup(schema( + ["users", "orders"], + [column("users", "id", 1, "bigint", 1), column("orders", "id", 1, "bigint", 1), + column("orders", "user_id", 2, "bigint", null)], + [{ + constraintName: "orders_user_id_fkey", sourceTableName: "orders", targetTableName: "users", + updateRule: "NO ACTION", deleteRule: "NO ACTION", deferrable: false, initiallyDeferred: false, + columns: [{ position: 1, sourceColumnName: "user_id", targetColumnName: "id" }], + }], + )); + await expect(physical.service.rebuildGenerated(physical.database.id)).resolves.toEqual({ + added: 0, alreadyPresent: 1, excluded: 0, ambiguous: 0, + }); + await expect(physical.service.addManual( + physical.database.id, + physical.endpoint("orders", "user_id").columnId, + physical.endpoint("users", "id").columnId, + )).rejects.toBeInstanceOf(LogicalRelationshipDuplicateError); +}); + +test("validates target uniqueness and canonical type compatibility for manual relationships", async () => { + const { database, service, endpoint } = await setup(schema( + ["users", "composite", "events"], + [column("users", "id", 1, "bigint", 1), + column("composite", "left_id", 1, "bigint", 1), column("composite", "right_id", 2, "bigint", 2), + column("events", "id", 1, "bigint", 1), column("events", "user_id", 2, "integer", null), + column("events", "composite_id", 3, "bigint", null)], + )); + await expect(service.addManual( + database.id, endpoint("events", "composite_id").columnId, endpoint("composite", "left_id").columnId, + )).rejects.toBeInstanceOf(LogicalRelationshipTargetNotUniqueError); + await expect(service.addManual( + database.id, endpoint("events", "user_id").columnId, endpoint("users", "id").columnId, + )).rejects.toBeInstanceOf(LogicalRelationshipTypeIncompatibleError); +}); + +test("rebuild is additive when an existing generated relationship stops matching", async () => { + const initial = schema( + ["users", "orders"], + [column("users", "id", 1, "bigint", 1), column("orders", "id", 1, "bigint", 1), + column("orders", "user_id", 2, "bigint", null)], + ); + const { repository, database, service } = await setup(initial); + await service.rebuildGenerated(database.id); + const changed = structuredClone(initial); + changed.columns.find((item) => item.tableName === "orders" && item.name === "user_id")!.dataType = "text"; + await repository.applySchemaSync(database.id, database.version, "columns", [], changed); + await expect(service.rebuildGenerated(database.id)).resolves.toEqual({ + added: 0, alreadyPresent: 0, excluded: 0, ambiguous: 0, + }); + expect((await service.list(database.id)).filter((item) => item.origin === "generated")).toHaveLength(1); +}); + +test("refuses inference until the current database version has a full schema sync", async () => { + const repository = new MemoryCatalogRepository(); + const database = await repository.create({ + workspaceId: "unsynced-relationships", + engine: "postgres", + databaseName: "warehouse", + schema: "public", + binding: { transport: "postgres_direct", host: "db.internal", port: 5432, username: "reader" }, + }); + const service = new CatalogLogicalRelationshipService(repository); + + await expect(service.rebuildGenerated(database.id)) + .rejects.toBeInstanceOf(LogicalRelationshipSchemaStaleError); +}); diff --git a/backend/test/catalog-repository.integration.test.ts b/backend/test/catalog-repository.integration.test.ts index 1a8559fa..9879cd0c 100644 --- a/backend/test/catalog-repository.integration.test.ts +++ b/backend/test/catalog-repository.integration.test.ts @@ -12,6 +12,7 @@ import { up as upRuntimeSequencePrivileges } from "../src/catalog/migrations/004 import { up as upDescriptionGeneration } from "../src/catalog/migrations/005_description_generation_runs.js"; import { up as upSensitiveDataFlag } from "../src/catalog/migrations/006_sensitive_data_flag.js"; import { up as upSensitiveSuggestionRuns } from "../src/catalog/migrations/007_sensitive_data_suggestion_runs.js"; +import { up as upLogicalRelationships } from "../src/catalog/migrations/008_catalog_logical_relationships.js"; const dockerAvailable = spawnSync("docker", ["info"], { stdio: "ignore" }).status === 0; @@ -26,6 +27,7 @@ test.skipIf(!dockerAvailable)("PostgreSQL migration enforces one database per wo await upTables(db); await upSchemaSync(db); await upSensitiveDataFlag(db); + await upLogicalRelationships(db); await sql`CREATE ROLE thothii_catalog_runtime`.execute(db); await upRuntimeSequencePrivileges(db); const sequencePrivilege = await sql<{ allowed: boolean }>` @@ -202,6 +204,7 @@ test.skipIf(!dockerAvailable)("PostgreSQL repository performs scoped metadata cl await upTables(db); await upSchemaSync(db); await upSensitiveDataFlag(db); + await upLogicalRelationships(db); const repository = new KyselyCatalogRepository(db); const database = await repository.create({ workspaceId: "cleanup-test", @@ -280,6 +283,7 @@ test.skipIf(!dockerAvailable)("PostgreSQL repository atomically consolidates sel await upTables(db); await upSchemaSync(db); await upSensitiveDataFlag(db); + await upLogicalRelationships(db); const repository = new KyselyCatalogRepository(db); const database = await repository.create({ workspaceId: "consolidation-test", @@ -379,6 +383,7 @@ test.skipIf(!dockerAvailable)("PostgreSQL repository persists description and se await upTables(db); await upSchemaSync(db); await upSensitiveDataFlag(db); + await upLogicalRelationships(db); await upDescriptionGeneration(db); await upSensitiveSuggestionRuns(db); const repository = new KyselyCatalogRepository(db); @@ -610,3 +615,63 @@ test.skipIf(!dockerAvailable)("PostgreSQL repository persists description and se await container.stop(); } }, 60_000); + +test.skipIf(!dockerAvailable)("PostgreSQL repository persists logical relationship lifecycle and tombstones", async () => { + const container = await new PostgreSqlContainer("postgres:17.6-bookworm").start(); + const db = new Kysely({ + dialect: new PostgresDialect({ pool: new Pool({ connectionString: container.getConnectionUri() }) }), + plugins: [new CamelCasePlugin()], + }); + try { + await upDatabases(db); + await upTables(db); + await upSchemaSync(db); + await upSensitiveDataFlag(db); + await upLogicalRelationships(db); + const repository = new KyselyCatalogRepository(db); + const database = await repository.create({ + workspaceId: "logical-relationships", + engine: "postgres", + databaseName: "warehouse", + schema: "public", + binding: { transport: "postgres_direct", host: "db.internal", port: 5432, username: "reader" }, + }); + await repository.applySchemaSync(database.id, database.version, "all", [], { + schemaVersion: 1, + capabilities: { tables: "available", columns: "available", relationships: "available" }, + tables: [{ name: "users", sourceComment: null }, { name: "orders", sourceComment: null }], + columns: [ + { tableName: "users", name: "id", ordinalPosition: 1, dataType: "bigint", isNullable: false, defaultExpression: null, primaryKeyPosition: 1, sourceComment: null }, + { tableName: "orders", name: "id", ordinalPosition: 1, dataType: "bigint", isNullable: false, defaultExpression: null, primaryKeyPosition: 1, sourceComment: null }, + { tableName: "orders", name: "user_id", ordinalPosition: 2, dataType: "bigint", isNullable: false, defaultExpression: null, primaryKeyPosition: null, sourceComment: null }, + ], + relationships: [], + }); + const context = (await repository.getLogicalRelationshipContext(database.id))!; + const source = context.endpoints.find((item) => item.tableName === "orders" && item.columnName === "user_id")!; + const target = context.endpoints.find((item) => item.tableName === "users" && item.columnName === "id")!; + const created = await repository.insertLogicalRelationship(database.id, source.columnId, target.columnId, false); + expect(created).toMatchObject({ origin: "manual", status: "active" }); + await expect(repository.insertLogicalRelationship(database.id, source.columnId, target.columnId, true)) + .resolves.toBeUndefined(); + + expect(await repository.setLogicalRelationshipStatus(database.id, created!.id, "excluded")) + .toMatchObject({ status: "excluded" }); + await expect(repository.insertGeneratedLogicalRelationships(database.id, [{ + sourceColumnId: source.columnId, targetColumnId: target.columnId, + }])).resolves.toBe(0); + + await expect(repository.deleteLogicalRelationship(database.id, created!.id)).resolves.toBe(true); + await expect(repository.insertGeneratedLogicalRelationships(database.id, [{ + sourceColumnId: source.columnId, targetColumnId: target.columnId, + }])).resolves.toBe(1); + expect(await repository.listLogicalRelationships(database.id)) + .toMatchObject([{ origin: "generated", status: "active" }]); + + await db.deleteFrom("catalogColumns").where("id", "=", source.columnId).execute(); + expect(await repository.listLogicalRelationships(database.id)).toEqual([]); + } finally { + await db.destroy(); + await container.stop(); + } +}, 60_000); diff --git a/backend/test/catalog-schema-routes.test.ts b/backend/test/catalog-schema-routes.test.ts index d7a54aaf..862c5dff 100644 --- a/backend/test/catalog-schema-routes.test.ts +++ b/backend/test/catalog-schema-routes.test.ts @@ -122,6 +122,7 @@ async function setup(env: Record = {}) { list: vi.fn(async () => [revision]), listCatalog: vi.fn(async () => [{ id: "psd-clinical", name: "Policlinico San Donato", configurationState: "ready", revision }]), read: vi.fn(async () => ({ workspace, revision })), + readPinned: vi.fn(async () => ({ workspace, workspaceConfigPath: revision.snapshotPath })), } as unknown as WorkspaceRegistry; const app = buildApp(loadConfig({ THT_HARNESS_DIR: "/missing", NODE_ENV: "test", ...env }), { thtRunner: {} as never, @@ -605,3 +606,125 @@ test("rejects a missing database without partially cleaning valid selections", a expect(response.statusCode).toBe(404); expect(await repository.listTables(database.id)).toHaveLength(2); }); + +test("serves one relationship map and supports the manual relationship lifecycle", async () => { + const { app, repository, database } = await setup(); + await seedCatalog(repository, database); + const tables = await repository.listTables(database.id); + const patients = tables.find((table) => table.name === "patients")!; + const visits = tables.find((table) => table.name === "visits")!; + const patientId = (await repository.listColumns(database.id, patients.id)).find((item) => item.name === "id")!; + const visitId = (await repository.listColumns(database.id, visits.id)).find((item) => item.name === "id")!; + + const created = await app.inject({ + method: "POST", + url: `/catalog/databases/${database.id}/relationships`, + payload: { sourceColumnId: visitId.id, targetColumnId: patientId.id }, + }); + expect(created.statusCode).toBe(201); + expect(created.json()).toMatchObject({ origin: "manual", status: "active", constraintName: null }); + + const listed = (await app.inject({ + method: "GET", url: `/catalog/databases/${database.id}/relationships`, + })).json(); + expect(listed.map((item: { origin: string }) => item.origin).sort()).toEqual(["manual", "physical"]); + + const relationshipId = created.json().id; + const excluded = await app.inject({ + method: "PATCH", + url: `/catalog/databases/${database.id}/relationships/${relationshipId}`, + payload: { status: "excluded" }, + }); + expect(excluded.statusCode).toBe(200); + expect(excluded.json()).toMatchObject({ origin: "manual", status: "excluded" }); + + const restored = await app.inject({ + method: "PATCH", + url: `/catalog/databases/${database.id}/relationships/${relationshipId}`, + payload: { status: "active" }, + }); + expect(restored.json()).toMatchObject({ status: "active" }); + + expect((await app.inject({ + method: "DELETE", url: `/catalog/databases/${database.id}/relationships/${relationshipId}`, + })).statusCode).toBe(204); +}); + +test("rejects generated inference while the Catalog schema is not current", async () => { + const { app, database } = await setup(); + + const response = await app.inject({ + method: "POST", + url: `/catalog/databases/${database.id}/relationships/rebuild-generated`, + }); + + expect(response.statusCode).toBe(409); + expect(response.json()).toEqual({ + code: "relationship_schema_stale", + message: "Synchronize the current database schema before managing logical relationships.", + }); +}); + +test("rebuilds generated relationships and returns the exact summary", async () => { + const { app, repository, database } = await setup(); + const observed = snapshot(); + observed.relationships = []; + await seedCatalog(repository, database, observed); + + const first = await app.inject({ + method: "POST", url: `/catalog/databases/${database.id}/relationships/rebuild-generated`, + }); + expect(first.statusCode).toBe(200); + expect(first.json()).toEqual({ added: 1, alreadyPresent: 0, excluded: 0, ambiguous: 0 }); + const generated = (await repository.listLogicalRelationships(database.id))[0]!; + + await app.inject({ + method: "PATCH", + url: `/catalog/databases/${database.id}/relationships/${generated.id}`, + payload: { status: "excluded" }, + }); + const second = await app.inject({ + method: "POST", url: `/catalog/databases/${database.id}/relationships/rebuild-generated`, + }); + expect(second.json()).toEqual({ added: 0, alreadyPresent: 0, excluded: 1, ambiguous: 0 }); +}); + +test("validates relationship requests and keeps physical relationships read-only", async () => { + const { app, repository, database } = await setup(); + await seedCatalog(repository, database); + const physical = (await repository.listRelationships(database.id))[0]!; + + const invalid = await app.inject({ + method: "POST", + url: `/catalog/databases/${database.id}/relationships`, + payload: { sourceColumnId: physical.columns[0].sourceColumnId, targetColumnId: physical.columns[0].targetColumnId, generated: true }, + }); + expect(invalid.statusCode).toBe(400); + expect(invalid.json()).toMatchObject({ code: "relationship_request_invalid" }); + + const readOnly = await app.inject({ + method: "PATCH", + url: `/catalog/databases/${database.id}/relationships/${physical.id}`, + payload: { status: "excluded" }, + }); + expect(readOnly.statusCode).toBe(409); + expect(readOnly.json()).toMatchObject({ code: "relationship_read_only" }); +}); + +test("requires database.manage for relationship map mutations", async () => { + const { app, repository, database } = await setup({ AUTH_MODE: "upstream" }); + const observed = snapshot(); + observed.relationships = []; + await seedCatalog(repository, database, observed); + const response = await app.inject({ + method: "POST", + url: `/catalog/databases/${database.id}/relationships/rebuild-generated`, + headers: { + "x-thoth-principal-issuer": "portal", + "x-thoth-principal-subject": "catalog-reader", + "x-thoth-is-admin": "0", + }, + }); + expect(response.statusCode).toBe(403); + expect(response.json()).toEqual({ code: "auth_forbidden", error: "This operation is not permitted" }); +}); diff --git a/backend/test/catalog-tables-routes.test.ts b/backend/test/catalog-tables-routes.test.ts index f787f6d1..3bac80ff 100644 --- a/backend/test/catalog-tables-routes.test.ts +++ b/backend/test/catalog-tables-routes.test.ts @@ -54,6 +54,7 @@ async function setup() { list: vi.fn(async () => [revision]), listCatalog: vi.fn(async () => [{ id: "psd-clinical", name: "Policlinico San Donato", configurationState: "ready", revision }]), read: vi.fn(async () => ({ workspace, revision })), + readPinned: vi.fn(async () => ({ workspace, workspaceConfigPath: revision.snapshotPath })), } as unknown as WorkspaceRegistry; const secretStore = new WorkspaceSecretStore({ root: secretRoot, runtimeRoot, installationId: "test" }); const app = buildApp(loadConfig({ THT_HARNESS_DIR: "/missing", NODE_ENV: "test" }), { diff --git a/backend/test/effective-relationship-snapshot.test.ts b/backend/test/effective-relationship-snapshot.test.ts new file mode 100644 index 00000000..72d82fa3 --- /dev/null +++ b/backend/test/effective-relationship-snapshot.test.ts @@ -0,0 +1,196 @@ +import { describe, expect, test, vi } from "vitest"; +import { EffectiveRelationshipSnapshotProvider } from "../src/catalog/effective-relationship-snapshot.js"; +import type { CatalogRelationship, WorkspaceDatabase } from "../src/catalog/types.js"; + +const timestamp = "2026-08-31T10:00:00.000Z"; + +function database(): WorkspaceDatabase { + return { + id: "database-1", + workspaceId: "psd", + engine: "postgres", + databaseName: "warehouse", + schema: "public", + version: 1, + createdAt: timestamp, + updatedAt: timestamp, + binding: { transport: "postgres_direct" }, + connectionStatus: "reachable", + schemaSyncedVersion: 1, + schemaSyncedAt: timestamp, + }; +} + +const operations = { + async run(_databaseId: string, operation: () => Promise): Promise { + return await operation(); + }, +}; + +function relationship(options: { + id: string; + origin: "physical" | "generated" | "manual"; + status?: "active" | "excluded"; + sourceTable?: string; + sourceColumns?: string[]; + targetTable?: string; + targetColumns?: string[]; +}): CatalogRelationship { + const sourceColumns = options.sourceColumns ?? ["user_id"]; + const targetColumns = options.targetColumns ?? ["id"]; + const columns = sourceColumns.map((sourceColumnName, position) => ({ + position: position + 1, + sourceColumnId: `source-${position}`, + sourceColumnName, + targetColumnId: `target-${position}`, + targetColumnName: targetColumns[position], + })); + const common = { + id: options.id, + databaseId: "database-1", + sourceTableId: "source-table", + sourceTableName: options.sourceTable ?? "orders", + targetTableId: "target-table", + targetTableName: options.targetTable ?? "users", + columns, + createdAt: timestamp, + updatedAt: timestamp, + }; + if (options.origin === "physical") { + return { + ...common, + constraintName: `fk_${options.id}`, + updateRule: "NO ACTION", + deleteRule: "NO ACTION", + deferrable: false, + initiallyDeferred: false, + lastSyncedDatabaseVersion: 1, + lastSyncedAt: timestamp, + origin: "physical", + status: "active", + }; + } + return { + ...common, + constraintName: null, + updateRule: null, + deleteRule: null, + deferrable: false, + initiallyDeferred: false, + lastSyncedDatabaseVersion: null, + lastSyncedAt: null, + origin: options.origin, + status: options.status ?? "active", + columns: [columns[0]!], + }; +} + +describe("EffectiveRelationshipSnapshotProvider", () => { + test("returns no projection for a workspace without a Catalog database", async () => { + const list = vi.fn(); + const provider = new EffectiveRelationshipSnapshotProvider( + { + get: vi.fn().mockResolvedValue(undefined), + getByWorkspace: vi.fn().mockResolvedValue(undefined), + }, + { list }, + operations, + ); + + await expect(provider.render("legacy")).resolves.toBeUndefined(); + expect(list).not.toHaveBeenCalled(); + }); + + test("renders a deterministic active map with physical precedence", async () => { + const duplicateGenerated = relationship({ id: "generated", origin: "generated" }); + const physical = relationship({ id: "physical", origin: "physical" }); + const compositePhysical = relationship({ + id: "physical-composite", + origin: "physical", + sourceTable: "order_lines", + sourceColumns: ["order_id", "tenant_id"], + targetTable: "orders", + targetColumns: ["id", "tenant_id"], + }); + const manual = relationship({ + id: "manual", + origin: "manual", + sourceTable: "invoices", + sourceColumns: ["customer_id"], + targetTable: "customers", + targetColumns: ["id"], + }); + const excluded = relationship({ + id: "excluded", + origin: "generated", + status: "excluded", + sourceTable: "invoices", + }); + const list = vi.fn().mockResolvedValue([ + manual, + duplicateGenerated, + excluded, + physical, + compositePhysical, + ]); + const provider = new EffectiveRelationshipSnapshotProvider( + { + get: vi.fn().mockResolvedValue(database()), + getByWorkspace: vi.fn().mockResolvedValue(database()), + }, + { list }, + operations, + ); + + const rendered = await provider.render("psd"); + expect(rendered?.endsWith("\n")).toBe(true); + expect(JSON.parse(rendered!)).toEqual({ + schemaVersion: 1, + workspaceId: "psd", + relationships: [ + { + sourceTable: "invoices", + sourceColumns: ["customer_id"], + targetTable: "customers", + targetColumns: ["id"], + origin: "manual", + }, + { + sourceTable: "order_lines", + sourceColumns: ["order_id", "tenant_id"], + targetTable: "orders", + targetColumns: ["id", "tenant_id"], + origin: "physical", + }, + { + sourceTable: "orders", + sourceColumns: ["user_id"], + targetTable: "users", + targetColumns: ["id"], + origin: "physical", + }, + ], + }); + expect(list).toHaveBeenCalledWith("database-1"); + }); + + test("fails closed when the Catalog schema is absent or stale", async () => { + const stale = database(); + delete stale.schemaSyncedVersion; + delete stale.schemaSyncedAt; + const list = vi.fn(); + const provider = new EffectiveRelationshipSnapshotProvider( + { + get: vi.fn().mockResolvedValue(stale), + getByWorkspace: vi.fn().mockResolvedValue(stale), + }, + { list }, + operations, + ); + + await expect(provider.render("psd")).rejects.toThrow( + "effective relationship snapshot requires a current full schema synchronization", + ); + expect(list).not.toHaveBeenCalled(); + }); +}); diff --git a/backend/test/routes-sessions.test.ts b/backend/test/routes-sessions.test.ts index 60e23cdb..1e170f69 100644 --- a/backend/test/routes-sessions.test.ts +++ b/backend/test/routes-sessions.test.ts @@ -517,6 +517,63 @@ test("creates a session from the active immutable workspace revision", async () })); }); +test("hands one effective relationship snapshot to both retrieval and Pi", async () => { + const effective = JSON.stringify({ + schemaVersion: 1, + workspaceId: "default", + relationships: [], + }); + const render = vi.fn(async () => effective); + const acquireWorkspaceRuntime = vi.fn((_workspace: string, relationships?: string) => ({ + path: "/runtime/with-relationships.yaml", + workspaceId: "default", + workspaceRevision: "e".repeat(40), + release: vi.fn(), + })); + const searchPack = vi.fn(async () => {}); + const createFor = vi.fn(() => ({ bridge: { onClientEvent: () => {} } })); + const app = buildApp(loadConfig({ THT_HARNESS_DIR: "../harness" }), { + thtRunner: { + sessionNew: async () => ({ id: "effective-map" }), + acquireWorkspaceRuntime, + searchPack, + } as any, + effectiveRelationshipSnapshotProvider: { render } as any, + readiness: { ensure: async () => ({ ok: true }) } as any, + mgr: { + get: () => undefined, + createFor, + configure: async () => {}, + start: () => {}, + } as any, + getSettings: () => ({ workspace: "default", thinking: "low" }) as any, + }); + + const response = await app.inject({ + method: "POST", + url: "/sessions", + payload: { question: "Which users placed orders?", workspaceId: "default" }, + }); + + expect(response.statusCode).toBe(200); + expect(render).toHaveBeenCalledWith("default"); + expect(acquireWorkspaceRuntime).toHaveBeenCalledWith( + expect.stringContaining("/default.yaml"), + effective, + ); + expect(createFor).toHaveBeenCalledWith( + "effective-map", + expect.objectContaining({ + runtimeConfig: expect.objectContaining({ path: "/runtime/with-relationships.yaml" }), + }), + ); + expect(searchPack).toHaveBeenCalledWith( + "Which users placed orders?", + "effective-map", + "/runtime/with-relationships.yaml", + ); +}); + test("rejects an SSH-only workspace before persisting or starting a session", async () => { const sessionNew = vi.fn(async () => ({ id: "must-not-exist" })); const ensure = vi.fn(async () => ({ ok: true })); diff --git a/backend/test/workspace-runtime-handoff.test.ts b/backend/test/workspace-runtime-handoff.test.ts index 5f29c7ab..d389b40c 100644 --- a/backend/test/workspace-runtime-handoff.test.ts +++ b/backend/test/workspace-runtime-handoff.test.ts @@ -197,6 +197,25 @@ test("ThtRunner uses a vault secret only for the lifetime of its runtime lease", expect(existsSync(rendered.database.password_file)).toBe(false); }); +test("ThtRunner binds and cleans the effective relationship snapshot with its runtime lease", async () => { + const f = await fixture(); + const runner = runnerFor(f); + const relationships = JSON.stringify({ + schemaVersion: 1, + workspaceId: "psd-clinical", + relationships: [], + }); + + const lease = runner.acquireWorkspaceRuntime(f.revision.snapshotPath, relationships); + const rendered = parse(readFileSync(lease.path, "utf8")) as { + paths: { effective_relationships: string }; + }; + + expect(readFileSync(rendered.paths.effective_relationships, "utf8")).toBe(relationships); + lease.release(); + expect(existsSync(rendered.paths.effective_relationships)).toBe(false); +}); + test("separate runtime leases hand off byte-identical revision Evidence configs accepted by tht", async () => { const f = await fixture(); const runner = runnerFor(f); diff --git a/docs/adr/0012-use-the-catalog-as-the-logical-relationship-authority.md b/docs/adr/0012-use-the-catalog-as-the-logical-relationship-authority.md new file mode 100644 index 00000000..a1d3d564 --- /dev/null +++ b/docs/adr/0012-use-the-catalog-as-the-logical-relationship-authority.md @@ -0,0 +1,36 @@ +# Use the Catalog as the logical relationship authority + +ThothII stores database-declared foreign keys and user-managed Logical Relationships in separate +Catalog models, as required by ADR-0006, but exposes them through one effective relationship map. +Physical relationships remain read-only and are refreshed from the database. Logical relationships +are either Generated by deterministic column-name inference or Manual; inference does not call an +AI model and does not inspect source values. + +A generated rebuild is additive. It preserves active and Manual relationships, never reactivates a +logically deleted relationship, and may recreate a relationship only after permanent deletion. A +logical deletion is therefore represented by retaining the relationship with an exclusion marker; +a permanent deletion removes it. Inference accepts only unambiguous, type-compatible matches to a +single-column primary key and skips all other candidates. It recognizes normalized table-qualified +names such as `user_id -> users.id`, exact non-generic primary-key names with one owner, and the +warehouse convention `*time_key -> dim_time.`. A source column may participate in a +composite primary key; bare generic names such as `id`, `key`, `code`, and `pk` are not evidence by +themselves. + +An exclusion is durable while both Catalog Column endpoints exist. Explicit metadata cleanup of an +endpoint table or column is a destructive boundary: it permanently removes every relationship and +exclusion attached to that endpoint, invalidates the synchronized-catalog marker, and requires a +full schema synchronization. The newly imported endpoints may then be inferred again. Preserving an +exclusion across endpoint destruction would require a second denormalized name-based identity, which +this design deliberately avoids. + +The installation-local Catalog is the sole writable authority for Logical Relationships. Git-pinned +workspace annotations remain authoritative for descriptive metadata but their embedded foreign keys +are legacy compatibility data. The backend materializes the active effective map as an immutable, +deterministic runtime snapshot when a session starts or resumes. When that snapshot is present, the +harness uses it as the exclusive relationship source and ignores relationships embedded in both the +physical schema artifact and workspace annotations; a missing or invalid declared snapshot fails +closed. Legacy runtimes without a snapshot retain the previous merge behavior. + +The snapshot is a projection, not another authored store. Its lifetime is tied to the runtime-config +lease, physical relationships take precedence over duplicate Logical Relationships, composite-key +column order is retained, and one Pi process observes one stable map for its complete lifetime. diff --git a/docs/operations/database-management.md b/docs/operations/database-management.md index af2a86cc..8b0fa90c 100644 --- a/docs/operations/database-management.md +++ b/docs/operations/database-management.md @@ -6,7 +6,7 @@ session workflow. ## What the catalog owns -For each YAML workspace, an administrator may create at most one database configuration. It holds +For each YAML workspace, an administrator may configure at most one Metadata Catalog binding. It holds the database name, schema, connection binding, write-only encrypted secrets, observed physical schema, optional curated descriptions, generated descriptions, and durable operation history. @@ -27,6 +27,18 @@ It uses `GET /catalog/metrics` without `databaseId` for installation totals and the current database. Choose a selection-scoped operation from the action selector and then press **Run**; unavailable operations remain listed with an explanation. Row-specific actions are the icon controls in the final column, and each navigation or action icon has an immediate conceptual tooltip. +An unconfigured workspace exposes **Configure catalog** directly on its row; there is no global +database-creation action and the selected workspace cannot be changed in the configuration form. + +The master grid keeps three independent states visible: + +- **Revision / Evidence** comes from the active immutable workspace revision. Filesystem Evidence is + materialized with that revision; remote Evidence is reported as configured-but-unverified or as + requiring credentials. +- **NL→SQL runtime** is calculated from the workspace DWH/Evidence requirements and runtime secret + store. It also reports transports, such as SSH, that are diagnostic-only and unsupported by sessions. +- **Metadata Catalog** reports whether the installation-local catalog configuration exists, then shows + its separately versioned connection-test or synchronization state. Configuration, object details, metadata editors, synchronization history, description history, sensitive-field review, and suggestion-run history open in right-side drawers backed by the @@ -42,8 +54,9 @@ remains available only until the integrated Fleet Ledger surface passes owner ac ## Configure and test a database -1. Open **Database Management** and choose a workspace. -2. Create its PostgreSQL configuration. Choose `postgres_direct`, `rest_api`, or `ssh_tunnel` and +1. Open **Database Management** and find the repository workspace marked **Not configured**. +2. Choose **Configure catalog** on that row. Configure its PostgreSQL catalog binding with + `postgres_direct`, `rest_api`, or `ssh_tunnel` and complete the binding fields that the chosen transport requires. 3. Enter secrets only when replacing them. They remain write-only and are never returned by the application. diff --git a/docs/operations/docker-refresh.md b/docs/operations/docker-refresh.md new file mode 100644 index 00000000..0bc11a09 --- /dev/null +++ b/docs/operations/docker-refresh.md @@ -0,0 +1,43 @@ +# ThothII Docker refresh + +The active local `psd-local` stack uses the operator environment generated for the installation: + +`deploy/psd/operator.env` + +It is not `deploy/thothii.env` (that file is empty) and `deploy/env/local.env` is only an example +path referenced by the generic launcher. The running stack also uses these Compose overlays: + +```text +compose.yaml +deploy/compose.local.yaml +deploy/compose.git-ssh.yaml +deploy/psd/connector-secrets.yaml +``` + +Refresh the stack from the repository root with: + +```bash +compose_psd=( + docker compose + --env-file deploy/psd/operator.env + -p thothii-18998cca7b0a + -f compose.yaml + -f deploy/compose.local.yaml + -f deploy/compose.git-ssh.yaml + -f deploy/psd/connector-secrets.yaml +) + +"${compose_psd[@]}" config --quiet +"${compose_psd[@]}" build core frontend +"${compose_psd[@]}" stop core frontend +"${compose_psd[@]}" up -d catalog-db +"${compose_psd[@]}" run --rm catalog-migrate +"${compose_psd[@]}" up -d --remove-orphans +``` + +Building before the stop keeps the existing application available if an image fails to compile. +Stopping only `core` and `frontend` prevents the old backend from using a newly migrated catalog; +the database, Qdrant, embedding service, named volumes, and installation state remain in place. + +The installation secret files referenced by that env file live under `deploy/psd/secrets/` and +must never be committed or printed. diff --git a/docs/plans/2026-08-26-metadata-catalog-from-thothai.md b/docs/plans/2026-08-26-metadata-catalog-from-thothai.md index 2aafe5b3..6474313a 100644 --- a/docs/plans/2026-08-26-metadata-catalog-from-thothai.md +++ b/docs/plans/2026-08-26-metadata-catalog-from-thothai.md @@ -109,12 +109,13 @@ canonico; i percorsi e i comportamenti descrivono il sorgente disponibile il 202 conserva il valore esistente e la sostituzione è un'azione esplicita. Delete rimuove anche i segreti associati. 34. La pagina usa AG Grid come master e un form React come detail, con sezioni Database, Connection - e TLS/SSH condizionali. La toolbar offre `Add database`; le righe `unconfigured` offrono - `Configure`. Entrambe selezionano esclusivamente workspace YAML senza un database e creano il - record soltanto al Save; `workspace_id` diventa immutabile dopo la creazione. -35. La grid mostra workspace, database, schema, transport, endpoint, stato connessione e ultimo - aggiornamento. Su schermi piccoli il dettaglio occupa il pannello completo. Il cambio riga con - modifiche non salvate e Delete richiedono conferma, senza conferma testuale tipizzata. + e TLS/SSH condizionali. Non esiste un'azione globale `Add database`: ogni riga `unconfigured` + offre `Configure catalog`, apre il form già vincolato a quello specifico workspace YAML e crea il + record soltanto al Save; `workspace_id` non è selezionabile né modificabile. +35. La grid mostra separatamente revisione/Evidence del workspace, binding runtime NL→SQL e + configurazione del Metadata Catalog, oltre a database, schema, endpoint e ultimo aggiornamento. + Su schermi piccoli il dettaglio occupa il pannello completo. Il cambio riga con modifiche non + salvate e Delete richiedono conferma, senza conferma testuale tipizzata. 36. La Database Binding conserva `connection_status`, `tested_version`, `last_tested_at`, un codice errore e un messaggio breve sanificato. Non conserva stack trace, DSN, credenziali o output grezzo del driver. diff --git a/docs/research/2026-08-31-browser-erd-library-options.md b/docs/research/2026-08-31-browser-erd-library-options.md new file mode 100644 index 00000000..ebe00016 --- /dev/null +++ b/docs/research/2026-08-31-browser-erd-library-options.md @@ -0,0 +1,299 @@ +# Visualizzatore ERD nel browser: proposta solo open source + +Data dell'audit: 31 agosto 2026. + +## Decisione + +Per ThothII sceglierei **AntV X6 + ELK.js** come fondazione del visualizzatore ERD. + +- **AntV X6** è MIT, non ha una distinta edizione Professional e include nel progetto OSS + le funzioni che servono davvero a un diagramma complesso: nodi e porte personalizzabili, + pan/zoom, selezione, minimappa, history, clipboard, tastiera, routing, virtual rendering ed + export SVG/PNG/JPEG. +- **ELK.js** è il motore di layout automatico, EPL-2.0 e senza tier commerciale. Il layout + layered gestisce porte, vincoli sull'ordine delle porte, archi multipli, self-loop e routing + ortogonale: proprietà importanti per foreign key composite e schemi affollati. +- La combinazione non dipende da esempi o componenti a pagamento. Il lavoro applicativo che + resta da fare — semantica ER, livelli di dettaglio, filtri e adattatore X6/ELK — appartiene + davvero al dominio di ThothII e non è una funzione nascosta dietro un piano Pro. + +La seconda scelta è **React Flow + ELK.js**. È probabilmente l'integrazione più naturale con +l'attuale frontend React e ha una migliore storia documentata per accessibilità. Il runtime è +MIT e non viene tecnicamente depotenziato, ma diversi pattern avanzati già implementati +(auto-layout, edge routing, undo/redo, copy/paste, expand/collapse) sono pubblicati come esempi +con licenza React Flow Pro. Non è quindi la scelta più lineare se il criterio prioritario è +evitare anche una dipendenza progettuale da materiale Professional. + +## Cosa significa «open source senza limitazioni Professional» + +L'audit distingue tre casi: + +1. **OSS completo**: licenza OSI e nessun tier commerciale che trattenga funzioni essenziali. +2. **OSS con caveat**: il motore è aperto, ma esistono esempi premium, API instabili o limiti + architetturali rilevanti. Può essere usato, purché il limite sia accettato esplicitamente. +3. **Escluso**: licenza proprietaria oppure core aperto con funzioni necessarie al viewer + complesso riservate alla versione commerciale. + +La valutazione riguarda sia la licenza sia ciò che si può effettivamente costruire senza +acquistare un prodotto complementare. Le note sulle licenze sono tecniche, non consulenza legale. + +## Matrice di selezione + +| Soluzione | Licenza OSS | Tier Pro rilevante | Adeguatezza a ERD complessi | Verdetto | +|---|---|---|---|---| +| **AntV X6 + ELK.js** | MIT + EPL-2.0 | Nessuno individuato | Porte per colonna, minimappa, routing, export, virtual rendering e layout avanzato | **Consigliata** | +| **React Flow + ELK.js** | MIT + EPL-2.0 | Esempi/pattern avanzati Pro, non runtime separato | Ottima UX React, nodi HTML, minimappa e accessibilità; export SVG non nativo | **Valida con caveat** | +| **maxGraph** | Apache-2.0 | Nessuno | Molto completo, SVG, porte, folding e layout; API imperativa e integrazione React costosa | **Valida con caveat** | +| **Cytoscape.js + ELK** | MIT | Nessuno | Molto scalabile come grafo, ma meno adatto a tabelle ricche e porte per campo | **Alternativa di nicchia** | +| **Liam ERD/CLI** | Apache-2.0 | Nessuno | Viewer ERD già rifinito e self-hostable; il package embeddable è interno e instabile | **Reference o app separata** | +| **Graphviz + Viz.js** | EPL-2.0 + MIT | Nessuno | Layout ed SVG eccellenti; non offre da solo un explorer applicativo | **Renderer di export** | +| **Mermaid.js + Panzoom** | MIT + MIT/ISC | Mermaid Chart è separato | Facile, ma insufficiente per interazioni e modelli ER ricchi | **Solo preview semplice** | +| **Sprotty/GLSP** | EPL-2.0 | Nessuno | Potente e completo, ma è un framework di modeling più pesante del necessario | **Non prioritario** | +| **JointJS** | Core MPL-2.0 | Molte funzioni utili sono in JointJS+ | Lo split commerciale intercetta proprio le necessità del viewer | **Esclusa** | +| **GoJS** | Proprietaria | Licenza di deployment | Completa tecnicamente, ma non open source | **Esclusa** | +| **yFiles** | Proprietaria | Licenza commerciale | Completa tecnicamente, ma non open source | **Esclusa** | + +## 1. Scelta principale: AntV X6 + ELK.js + +### Funzioni disponibili nell'open source + +X6 offre un canvas diagrammatico basato su SVG con supporto anche a nodi HTML/React. Il core +espone porte, archi personalizzabili, self-loop e multiedge. I plugin distribuiti dal progetto +comprendono Scroller, MiniMap, Selection, History, Clipboard, Keyboard, DnD, Stencil, Snapline, +Transform ed Export. La documentazione copre inoltre router `orth`, `manhattan` ed `er`, zoom e +panning. Non è emersa un'edizione X6 Pro che renda a pagamento queste capacità. + +Fonti primarie: [repository e licenza X6](https://github.com/antvis/X6), +[porte](https://x6.antv.antgroup.com/en/tutorial/basic/port), +[router](https://x6.antv.antgroup.com/en/api/registry/router), +[minimappa](https://x6.antv.antgroup.com/en/tutorial/plugins/minimap), +[scroller](https://x6.antv.antgroup.com/en/tutorial/plugins/scroller) ed +[export](https://x6.antv.antgroup.com/en/tutorial/plugins/export). + +ELK non è un renderer: calcola la geometria del grafo. L'algoritmo layered supporta porte e +relativi vincoli, archi ortogonali, self-loop e archi multipli. Va eseguito in un Web Worker per +non bloccare l'interfaccia sui modelli grandi. L'adattatore dovrà passare a ELK le porte delle +colonne e usare anche le `edge sections` e i bend point restituiti, non soltanto le coordinate +dei nodi. + +Fonti primarie: [ELK.js e licenza](https://github.com/kieler/elkjs) e +[ELK layered](https://eclipse.dev/elk/reference/algorithms/org-eclipse-elk-layered.html). + +### Limiti reali + +- X6 è più imperativo di React Flow: conviene incapsularlo in un singolo componente React con + un adapter stabile, evitando di spargere istanze e listener nel resto dell'applicazione. +- L'accessibilità per screen reader non è documentata al livello di React Flow; tab order, + focus, descrizioni ARIA e navigazione da tastiera richiedono test e lavoro applicativo. +- Un nodo React/HTML può introdurre `foreignObject` nell'SVG. Per un export portabile e + stampabile è meglio produrre un SVG separato dallo stesso modello e dalla geometria ELK. + +Questi sono costi tecnici, non limitazioni commerciali. + +## 2. Seconda scelta: React Flow + ELK.js + +`@xyflow/react` è MIT. Custom nodes React, handle multipli, pan/zoom, fit view, Controls, +MiniMap, selezione e funzioni di accessibilità sono nel runtime OSS. React Flow Pro vende +supporto, template ed esempi con relativo codice sorgente; non esiste una libreria runtime Pro +che sblocchi il canvas. + +Il caveat è comunque concreto: auto-layout, edge routing, undo/redo, copy/paste ed +expand/collapse compaiono nel catalogo degli esempi Pro e quel codice usa la **xyflow Pro +License**, non la MIT del core. Le stesse funzioni possono essere sviluppate sopra le API OSS, +ma non si può considerare tutto il materiale ufficiale liberamente riutilizzabile. + +Fonti primarie: [package React Flow](https://github.com/xyflow/xyflow/blob/main/packages/react/package.json), +[funzioni del core](https://reactflow.dev/index), +[catalogo degli esempi Pro](https://reactflow.dev/examples/pro-examples), +[auto-layout](https://reactflow.dev/examples/layout/auto-layout) e +[termini Pro](https://reactflow.dev/pro). + +È una buona scelta se si privilegiano velocità d'integrazione React, componenti HTML e +accessibilità. Non è la prima scelta di questo audit perché l'utente ha chiesto espressamente +di minimizzare la distanza fra ciò che è open source e l'offerta Professional. + +Altro limite: React Flow combina nodi DOM e archi SVG. L'esempio ufficiale di download usa +`html-to-image`, quindi il diagramma interattivo non si traduce automaticamente in un SVG puro. + +## 3. Altre soluzioni interamente open source + +### maxGraph + +maxGraph, successore TypeScript di mxGraph, è Apache-2.0 e non ha un piano Pro. Offre rendering +SVG, connection constraints/porte, loop e multigraph, layout, routing, outline, grouping e +folding. Può quindi sostenere un editor ERD ricco. + +Lo terrei come terza scelta: l'API è imperativa, non esiste un wrapper React ufficiale, la +virtualizzazione non è documentata e gran parte dell'accessibilità va costruita. Il progetto è +ancora pre-1.0. Il costo di integrazione e manutenzione sarebbe maggiore di X6. + +Fonti: [licenza e repository](https://github.com/maxGraph/maxGraph), +[guida Vite/TypeScript](https://maxgraph.github.io/maxGraph/docs/getting-started/) e +[gestione della complessità](https://maxgraph.github.io/maxGraph/docs/usage/group-and-complexity-management/). + +### Cytoscape.js + +Cytoscape.js e molte estensioni sono MIT, senza tier professionale. È ottimo per grandi grafi, +filtri, selezioni e algoritmi di rete. Il rendering Canvas è però meno naturale per schede-tabella +ricche, testo selezionabile e handle collegati alle singole colonne. Inoltre l'adapter +`cytoscape.js-elk` non passa le porte a ELK né usa le route degli archi: non risolve da solo il +problema delle foreign key per colonna. + +È appropriato per una vista di dipendenze ad alto livello, non come renderer ERD principale. +L'estensione `cytoscape-svg` è GPL-3.0; in un prodotto che non vuole assorbire quel vincolo è +preferibile una pipeline Graphviz/Viz.js o un generatore SVG proprio. + +Fonti: [Cytoscape.js](https://github.com/cytoscape/cytoscape.js), +[adapter ELK](https://github.com/cytoscape/cytoscape.js-elk) e +[cytoscape-svg](https://github.com/kinimesi/cytoscape-svg). + +### Liam ERD + +Liam ERD è Apache-2.0, self-hostable e già orientato al problema: pan/zoom, ricerca, filtro, +command palette e modalità `TABLE_NAME`, `KEY_ONLY`, `ALL_FIELDS`. La documentazione dichiara +supporto a schemi con oltre cento tabelle. La CLI può generare un'app Vite statica. + +Non userei però direttamente `@liam-hq/erd-core` dentro ThothII: il maintainer lo definisce una +dipendenza interna, ne sconsiglia l'uso diretto e avverte che l'API può cambiare; il package è +ancora 0.x. Inoltre occorre verificare la compatibilità con la versione React di ThothII e la +fedeltà delle foreign key composite. Liam è quindi un ottimo benchmark UX, una CLI per una vista +separata o una base da forkare consapevolmente, non l'interfaccia stabile su cui fondare il +prodotto. + +Fonti: [repository Liam](https://github.com/liam-hq/liam), +[README di erd-core](https://github.com/liam-hq/liam/blob/main/frontend/packages/erd-core/README.md), +[funzioni UI](https://liambx.com/docs/ui-features) e [CLI](https://liambx.com/docs/cli). + +### Sprotty/GLSP + +Sprotty e Eclipse GLSP sono EPL-2.0 e offrono un'infrastruttura seria per diagrammi SVG, +layout e protocolli client/server. Sono pensati per strumenti di modeling estensibili, con +dependency injection e un'architettura più ampia di un viewer. Restano una soluzione OSS valida, +ma sproporzionata per questa esigenza salvo che ThothII evolva in un vero editor di modelli. + +Fonti: [Sprotty](https://github.com/eclipse-sprotty/sprotty) ed +[Eclipse GLSP](https://eclipse.dev/glsp/documentation/overview/). + +## 4. Renderer complementari, non fondazioni del viewer + +### Graphviz e Viz.js + +Graphviz è EPL-2.0; `@viz-js/viz`, il port WebAssembly utilizzabile nel browser, è MIT. Graphviz +produce SVG di alta qualità, supporta label HTML-like e porte e rimane molto utile per export, +stampa o una vista read-only. Non fornisce però da solo ricerca, filtri, livelli di dettaglio, +editing o gestione dello stato applicativo. + +Lo userei come renderer di export alternativo, non come UI primaria. Se il layout a schermo +deve corrispondere esattamente all'export, è preferibile generare l'SVG direttamente dalla +geometria ELK invece di mantenere due motori di layout indipendenti. + +Fonti: [licenza Graphviz](https://graphviz.org/license/), +[formati SVG](https://graphviz.org/docs/outputs/svg/) e +[Viz.js](https://github.com/mdaines/viz-js). + +### Mermaid.js e librerie di pan/zoom + +Mermaid.js è MIT e non ha feature del renderer open source bloccate. **Mermaid Chart** è invece +un servizio commerciale distinto e il suo piano gratuito ha limiti: non va confuso con la +libreria self-hosted. + +Si può aggiungere navigazione a un SVG Mermaid con `@panzoom/panzoom` (MIT) o `d3-zoom` (ISC), +entrambi senza tier Pro. Questo migliora l'esperienza corrente, ma non supera i limiti strutturali +del diagramma ER Mermaid: interazioni a livello di tabella, controllo ridotto delle porte e del +routing, assenza di semantic zoom e difficoltà crescente su schemi molto grandi. + +Mermaid resta adatto a preview e documentazione, non al viewer finale. + +Fonti: [Mermaid.js](https://github.com/mermaid-js/mermaid), +[distinzione da Mermaid Chart](https://mermaid.ai/open-source/ecosystem/mermaid-chart.html), +[Panzoom](https://github.com/timmywil/panzoom) e [d3-zoom](https://github.com/d3/d3-zoom). + +### Python + +SchemaSpy ed ERAlchemy sono open source e possono generare documentazione o grafi attraverso +Graphviz, ma non sostituiscono il componente interattivo React. Python è utile lato backend per +normalizzare metadati o produrre artefatti batch; pan/zoom, selezione, ricerca e dettagli restano +responsabilità del browser. Aggiungere un servizio Python solo per disegnare l'ERD non porta un +vantaggio architetturale a ThothII. + +## 5. Soluzioni escluse + +### JointJS / JointJS+ + +Il core JointJS è MPL-2.0, ma JointJS+ è commerciale e comprende proprio molte funzioni che +servirebbero qui: PaperScroller, Navigator/minimappa, selection, clipboard, keyboard, undo/redo, +toolbar, export e layout aggiuntivi. Sarebbe tecnicamente possibile ricostruirle sul core, ma la +distanza fra OSS e Professional è troppo grande rispetto al criterio richiesto. + +Fonti: [licenza](https://www.jointjs.com/license), +[confronto delle funzioni](https://www.jointjs.com/features) e +[prezzi](https://www.jointjs.com/pricing). + +### GoJS e yFiles + +Sono prodotti completi e maturi, ma non open source. GoJS richiede una licenza per il deployment +e mostra una filigrana senza chiave; yFiles usa licenze proprietarie, con evaluation temporanea e +licenza necessaria per la distribuzione. Non soddisfano il requisito, quindi non entrano nella +shortlist. + +Fonti: [licensing GoJS](https://gojs.net/latest/intro/deployment.html) e +[licensing yFiles](https://docs.yworks.com/yfiles-html/dguide/deployment/licensing.html). + +## 6. Architettura proposta per ThothII + +Il frontend usa React 18 e oggi `SchemaLinkingViewer.tsx` genera Mermaid con un limite di 45 +elementi. La resa corrente perde informazione: accorpa più foreign key fra la stessa coppia di +tabelle, omette i self-reference e usa cardinalità generiche. Il catalogo espone già tabelle, +colonne, chiavi primarie e relazioni con coppie ordinate di colonne, quindi può alimentare un +modello più fedele. + +Propongo questa separazione: + +1. **Modello renderer-neutral**: `TableNode`, `ColumnPort` e `RelationEdge` con nome del + constraint e lista ordinata delle coppie sorgente/destinazione. Non appiattire le FK composite. +2. **Snapshot API**: un endpoint coerente, per esempio + `GET /catalog/databases/:databaseId/schema-diagram`, che restituisca tabelle, colonne, + relazioni e versione del catalogo in una sola lettura, evitando la richiesta colonne N+1. +3. **Layout worker**: ELK.js in Web Worker, con porte per colonna, port constraints, route degli + archi e cache per versione del database e filtri correnti. +4. **Renderer interattivo**: X6 incapsulato in `DatabaseRelationshipDiagram.tsx`, affiancato + alla vista tabellare con un toggle List/Diagram e con riuso del drawer dei dettagli. +5. **Export**: generatore SVG separato basato sullo stesso modello e sulla geometria ELK; + Graphviz/Viz.js può essere un fallback per layout alternativi o documentazione batch. + +Il catalogo non registra oggi tutti i vincoli UNIQUE né il tipo MATCH delle FK. Senza questi +dati non sempre è possibile distinguere correttamente 1:1 da 1:N. Il renderer non deve inventare +la cardinalità: deve mostrare una notazione neutra finché il metadato necessario non viene +raccolto. + +### Funzioni necessarie per schemi complessi + +- semantic zoom: solo nomi tabella, poi PK/FK, infine tutti i campi; +- ricerca e filtri per schema, tabella, colonna e tipo di relazione; +- modalità focus con vicinato a uno o due hop; +- evidenziazione della relazione e delle due colonne al passaggio/selezione; +- minimappa, fit selection, navigazione da tastiera e pannello dettagli; +- distinzione visiva di PK, FK, nullable, composite key, self-loop e relazioni parallele; +- layout automatico ricalcolabile, ma posizioni manuali persistibili; +- rendering progressivo o virtuale e fallback tabellare sempre disponibile; +- export dell'intero schema e dell'area filtrata in SVG/PNG. + +## 7. Proof of concept consigliato + +Un POC breve dovrebbe usare **X6 + ELK.js** su tre dataset sintetici: circa 30, 150 e 500 +tabelle, includendo FK composite, più FK tra la stessa coppia, self-loop, tabelle isolate e hub +ad alto grado. + +I criteri di accettazione dovrebbero misurare: + +- tempo di layout nel worker e tempo al primo frame interattivo; +- fluidità di pan/zoom e uso memoria sul caso da 500 tabelle; +- correttezza di porte, archi paralleli, self-loop e FK composite; +- leggibilità nelle tre soglie di semantic zoom; +- navigazione completa da tastiera e comportamento con screen reader; +- qualità e portabilità dell'SVG esportato; +- assenza di codice, esempi o componenti soggetti a licenza commerciale. + +Se X6 non raggiunge il livello di accessibilità richiesto senza un costo eccessivo, il confronto +finale va fatto con React Flow + lo stesso adapter ELK e lo stesso modello dati. In questo modo +si cambia renderer senza rifare API, semantica delle relazioni o pipeline di export. diff --git a/docs/research/2026-08-31-relationship-management-thothai-to-thothii.md b/docs/research/2026-08-31-relationship-management-thothai-to-thothii.md new file mode 100644 index 00000000..bdc68283 --- /dev/null +++ b/docs/research/2026-08-31-relationship-management-thothai-to-thothii.md @@ -0,0 +1,246 @@ +# Gestione delle relationship: da ThothAI a ThothII + +**Stato:** analisi e direzione funzionale/UX confermate nel *grill with docs*; non costituisce ancora un piano di implementazione. +**Revisione esaminata:** commit ThothII `f586152636b1fd653b0bc1d40b54be7f89bbd2bb`. I sorgenti legacy di ThothAI citati sotto sono versionati nello stesso repository, sotto `Thoth/ThothAI/`. +**Ambito:** import delle foreign key fisiche, inferenza di relationship logiche, modifica umana, pubblicazione verso il workflow NL→SQL e principali gap tra i due sistemi. + +## Sintesi fattuale + +1. In ThothAI le foreign key dichiarate nel database venivano importate e una procedura separata proponeva relationship basate sul nome dei campi e su una verifica dei valori. Entrambi i percorsi scrivevano però nello stesso modello `Relationship` (`Thoth/ThothAI/backend/thoth_core/dbmanagement.py:484-640`; `Thoth/ThothAI/backend/thoth_core/admin_models/admin_sqldb.py:571-826`; `Thoth/ThothAI/backend/thoth_core/models.py:488-530`). +2. Il ricordo di una relationship inferita persistita come `generated` non trova riscontro nel modello esaminato: `Relationship` contiene solo quattro foreign key verso tabelle e colonne, senza provenienza, stato, confidenza o flag `generated`. La procedura di inferenza calcola localmente pattern e tasso di validazione, ma non li salva (`Thoth/ThothAI/backend/thoth_core/models.py:488-530`; `Thoth/ThothAI/backend/thoth_core/admin_models/admin_sqldb.py:697-801`). +3. L'amministratore Django di ThothAI permetteva CRUD manuale sul medesimo insieme di relationship e verificava che gli endpoint appartenessero allo stesso database e alle tabelle selezionate; non distingueva visivamente o semanticamente relationship fisiche, inferite e manuali (`Thoth/ThothAI/backend/thoth_core/admin_models/admin_relationship.py:19-313`). +4. ThothII oggi separa già i due concetti: il catalogo backend conserva solo le foreign key fisiche dichiarate, mentre il core/harness possiede annotazioni di foreign key logiche curate e un comando che suggerisce candidate per nome, primary key e SQL osservato (`CONTEXT.md:291-300`; `docs/adr/0006-separate-physical-and-logical-relationships.md:3-8`; `harness/tht/cli/schema_cmd.py:203-299`). +5. I due mondi ThothII non sono ancora integrati: i record di Database Management non modificano il workflow NL→SQL e l'integrazione catalogo→core/schema-linking è ancora un gate di design esplicitamente differito (`PROJECT_STATE.md:116-124`; `PROJECT_STATE.md:161-165`). + +## Evidenze ThothAI + +### Foreign key ufficiali + +Il percorso di import legge le foreign key dal database, limita l'import alle tabelle già presenti nel catalogo, crea le colonne mancanti, crea o riusa un record `Relationship` e aggiorna anche le stringhe denormalizzate `pk_field`/`fk_field` sulle colonne (`Thoth/ThothAI/backend/thoth_core/dbmanagement.py:484-640`, in particolare `:501-513`, `:526-591` e `:606-607`). + +**Fatto:** una foreign key fisica diventa quindi un record applicativo, non rimane soltanto un fatto letto al momento dal database. + +### Relationship inferite + +La routine legacy dichiara sei famiglie di confronto tra il nome della colonna candidata e quello della primary key: corrispondenza esatta, snake case, kebab case, camel case, concatenazione e solo nome tabella (`Thoth/ThothAI/backend/thoth_core/admin_models/admin_sqldb.py:571-689`). + +Per una candidata, la routine: + +- legge fino a 20 valori distinti e non nulli dalla colonna candidata; +- verifica ogni valore contro la primary key bersaglio; +- accetta la candidata quando almeno il 70% dei valori esaminati trova riscontro; +- crea o recupera un normale `Relationship` e aggiorna i campi denormalizzati delle tabelle (`Thoth/ThothAI/backend/thoth_core/admin_models/admin_sqldb.py:697-801`). + +**Correzione documentale:** un commento parla di campionamento casuale, ma la query mostrata non contiene un ordinamento casuale; il comportamento verificabile dal codice è “fino a 20 valori distinti e non nulli”, non un campione statisticamente casuale (`Thoth/ThothAI/backend/thoth_core/admin_models/admin_sqldb.py:697-716`). + +**Rischio legacy:** nomi di tabelle e colonne sono interpolati direttamente in SQL in questo percorso. Portare la logica letteralmente in ThothII riprodurrebbe un problema di quoting/sicurezza e richiederebbe inoltre una policy esplicita per l'accesso ai valori del DWH (`Thoth/ThothAI/backend/thoth_core/admin_models/admin_sqldb.py:709-750`). + +### Un solo modello per tre origini + +Il modello `Relationship` legacy contiene soltanto `source_table`, `target_table`, `source_column` e `target_column`, più metodi di rappresentazione/aggiornamento. Non contiene campi per origine, algoritmo, evidenza, confidenza, approvazione o disabilitazione, né un vincolo di unicità dichiarato nel modello (`Thoth/ThothAI/backend/thoth_core/models.py:488-530`). + +L'admin consente aggiunta, modifica e cancellazione ordinarie e valida la coerenza tra database, tabelle e colonne (`Thoth/ThothAI/backend/thoth_core/admin_models/admin_relationship.py:30-38`, `:125-157`, `:218-245`). + +**Conclusione fattuale:** relationship importate, inferite e create manualmente convergono nello stesso tipo persistito. Dal record finale non è possibile ricostruirne con certezza l'origine. Il termine `generated`, se usato nell'interfaccia o nel linguaggio operativo dell'epoca, non era una qualificazione persistita dal modello esaminato. + +### Uso nella comprensione dello schema + +La generazione M-Schema conserva colonne PK/FK e ricostruisce la sezione `【Foreign keys】` analizzando le stringhe denormalizzate `fk_field` (`Thoth/ThothAI/frontend/sql_generator/helpers/main_helpers/main_generate_mschema.py:25-61`, `:94-101`, `:160-187`). + +**Fatto:** le relationship curate nel catalogo legacy influenzavano la rappresentazione dello schema consumata dal generatore SQL, anche se tramite una proiezione denormalizzata. + +## Stato attuale di ThothII + +### Catalogo backend: relationship fisiche + +Il modello di dominio corrente distingue esplicitamente: + +- **Physical Relationship:** vincolo dichiarato nel database; +- **Catalog Relationship:** copia persistita di quel fatto fisico, non creabile o modificabile manualmente ma eliminabile per cleanup; +- **Logical Relationship:** relazione semantica curata o inferita, con ownership e lifecycle separati (`CONTEXT.md:291-300`; `docs/adr/0006-separate-physical-and-logical-relationships.md:3-8`). + +La migrazione del catalogo crea `catalog_relationships` e le coppie ordinate in `catalog_relationship_columns`. L'identità della relazione fisica è `(source_table_id, constraint_name)`; non esistono campi di origine, stato o confidenza (`backend/src/catalog/migrations/003_catalog_schema_sync.ts:42-74`). Il tipo TypeScript è solo strutturale e le descrizioni generate riguardano esclusivamente tabelle o colonne (`backend/src/catalog/types.ts:123-152`). + +L'introspezione PostgreSQL legge soltanto `pg_constraint` con `contype = 'f'` e mantiene l'ordine delle coppie per le chiavi composite (`backend/src/catalog/schema-introspector.ts:110-171`; `backend/src/catalog/schema-introspector.ts:360-416`; `docs/contracts/catalog-schema-snapshot.md:40-53`). La sincronizzazione autoritativa crea, aggiorna o elimina le copie fisiche in base allo snapshot (`backend/src/catalog/repository.ts:1165-1218`; `backend/src/catalog/repository.ts:1316-1393`). + +Le API e la UI espongono lettura, sync e cleanup in massa, non CRUD logico per singola relationship (`backend/src/routes/catalog-schema.ts:25-37`; `backend/src/routes/catalog-schema.ts:134-150`; `frontend/src/api/catalog-databases.ts:452-489`; `frontend/src/shell/database-management/DatabaseRelationships.tsx:114-123`; `frontend/src/shell/database-management/DatabaseRelationships.tsx:160-225`). + +**Effetto rilevante:** il cleanup è intenzionalmente reversibile tramite una sync successiva e può lasciare temporaneamente un catalogo incompleto (`docs/adr/0008-allow-manual-catalog-metadata-cleanup.md:7-16`). Un test di integrazione ammette anche una relationship rimasta senza coppie di colonne dopo la cancellazione delle colonne catalogate (`backend/test/catalog-repository.integration.test.ts:194-264`, in particolare `:246-252`). Un futuro consumer non può quindi assumere che ogni stato intermedio del catalogo sia pubblicabile così com'è. + +### Core/harness: relationship logiche e suggerimenti + +Il modello M-Schema del core possiede già `TableAnnotation.foreign_keys`, descritte come foreign key logiche curate e unite alle foreign key fisiche (`harness/tht/mschema/models.py:35-39`; `harness/tht/mschema/models.py:76-84`). Il renderer fonde i due insiemi e li presenta insieme nella sezione `【Foreign keys】` (`harness/tht/mschema/render.py:9-20`; `harness/tht/mschema/render.py:46-80`). + +Il comando `schema suggest-fks` costruisce candidate da: + +- uguaglianze trovate in SQL, quando esattamente un lato è una primary key; +- convenzione speciale `*time_key → dim_time.`; +- stesso nome tra colonna e primary key a proprietario univoco; +- assunzioni esplicite per disambiguare; +- esclusione di nomi PK generici come `id`, `key` e `code` e dei proprietari ambigui (`harness/tht/cli/schema_cmd.py:203-299`; `harness/tht/mschema/fkmine.py:1-58`). + +Il comando può produrre un documento candidato e, con `--write`, aggiungere annotazioni mancanti in modo idempotente; il messaggio stesso chiede revisione manuale (`harness/tht/cli/schema_cmd.py:406-429`; `harness/tht/cli/schema_cmd.py:519-540`; `harness/tests/test_schema_fk_annotations.py:131-146`; `harness/tests/test_schema_fk_annotations.py:454-470`). + +**Fatto:** ThothII non parte da zero sull'inferenza. Possiede già un motore più conservativo, basato su PK univoche e SQL osservato, ma non conserva per ogni relazione origine, evidenza, frequenza o confidenza. Il miner conta le occorrenze internamente, ma il modello candidato non promuove quel conteggio a lifecycle persistito (`harness/tht/mschema/fkmine.py:1-58`; `harness/tht/mschema/models.py:35-39`). + +**Rischi strutturali già visibili:** + +- `ForeignKey` ammette liste di colonne, ma non valida che source e target abbiano la stessa cardinalità; il renderer usa `zip`, quindi una relazione malformata può essere troncata silenziosamente (`harness/tht/mschema/models.py:35-39`; `harness/tht/mschema/render.py:76-78`). +- il merge evita duplicati rispetto alle FK fisiche iniziali, ma non aggiorna l'insieme `seen` dopo aver aggiunto un'annotazione; due annotazioni logiche uguali possono sopravvivere al merge (`harness/tht/mschema/render.py:9-20`). +- il catalogo fisico supporta coppie composite ordinate, mentre le euristiche correnti e legacy sono sostanzialmente unary. La semantica delle candidate composite resta da decidere (`backend/src/catalog/migrations/003_catalog_schema_sync.ts:56-74`; `harness/tht/cli/schema_cmd.py:203-299`). + +### Revisione e pubblicazione correnti + +Le annotazioni canoniche del workspace sono un blob Git. Il flusso pubblico produce candidate, verifica le annotazioni e richiede un'accettazione umana esplicita dopo commit/push/pull; l'accettazione registra digest del candidato e delle annotazioni, revisione e blob (`docs/contracts/workspace-preprocessing-cli.md:89-106`; `backend/src/workspaces/preprocessing-service.ts:202-297`). Il preprocessing successivo procede solo se il digest accettato coincide con le annotazioni correnti (`backend/src/workspaces/preprocessing-service.ts:335-388`). + +**Limite fattuale:** il record di review conserva digest, revisione e blob, ma non attore, motivazione o decisioni per singola candidata (`backend/src/workspaces/preprocessing-state.ts:67-73`). + +Il runtime usa le annotazioni quando renderizza lo schema, ma la ricerca vettoriale indicizza record di tabelle e colonne senza contenuto esplicito delle relationship (`harness/tht/vectorstore/records.py:106-138`; `harness/tht/cli/search_cmd.py:233-273`). Inoltre la vista colonne usata in F4 continua a leggere i commenti fisici, non le annotazioni (`harness/tht/cli/schema_cmd.py:592-621`). + +La review dei join durante una sessione è distinta dalla curatela globale: il reviewer conferma l'insieme dei join oppure chiede una revisione completa; non modifica la mappa canonica delle relationship (`harness/.pi/skills/tht-sessione/SKILL.md:301-341`; `frontend/src/widgets/JoinReviewWidget.tsx:5-84`). Il modello di sessione registra join come due stringhe e una decisione opzionale, senza ID stabile della relationship o coppie ordinate strutturate (`harness/tht/session/models.py:147-170`). + +## Delta e rischi da sottoporre al grill + +| Tema | Fatto documentato | Delta/rischio aperto | +|---|---|---| +| Origine | ThothAI perdeva l'origine; ThothII separa fisico e logico a livello concettuale | Decidere quale provenienza debba essere persistita per manuale, euristica, SQL osservato e import fisico | +| `generated` | Non era un flag del modello ThothAI; in ThothII “Generated Description” è già un termine del catalogo (`CONTEXT.md:302-311`) | Usare `generated` anche per relationship potrebbe creare ambiguità terminologica | +| Cancellazione utente | In ThothAI era CRUD sul record unico; in ThothII il cleanup fisico viene ricostruito dalla sync | “Eliminare” può significare cancellare una relazione logica, sopprimere un fatto fisico per il core oppure pulire temporaneamente la copia catalogata: sono operazioni diverse | +| Inferenza | ThothAI usava sei pattern e valori DWH; ThothII usa PK univoche, nomi e SQL osservato | Stabilire se sostituire, integrare o non portare il campionamento dei valori; servono policy di dati sensibili, query read-only, quoting e limiti | +| Approvazione | ThothII dispone di review Git/digest dell'intero artefatto | Mancano decisioni per candidata, motivazione, attore, sticky rejection, versione algoritmo e gestione dello stale | +| Compositi | Il catalogo fisico conserva coppie ordinate; le annotazioni accettano liste | Mancano invarianti forti e una strategia di inferenza/review per join compositi | +| Pubblicazione | Catalogo management e runtime core sono oggi separati | Va stabilito se pubblicare tutto, una selezione esplicita o una revisione immutabile; il catalogo può essere incompleto durante cleanup/sync | +| UI e ownership | UI catalogo fisico read-only; curatela logica via CLI/Git; review join per sessione separata | Va scelto chi cura la mappa e in quale superficie, senza confondere amministrazione globale e correzione della singola sessione | +| Retrieval | Le relationship entrano nel render M-Schema ma non nei record vettoriali | Va deciso se e come influenzano selezione tabelle, ranking e descrizioni, oltre al rendering finale | +| Drift | Sync fisica è autoritativa; annotazioni sono una revisione separata | Servono semantiche per endpoint rinominati/eliminati, candidate stale, orphan e riapprovazione | + +Le analisi già presenti nel repository trattano l'integrazione catalogo→core, la selezione pubblicabile e la riparazione degli indici come questioni ancora aperte; le loro proposte non sono decisioni implementate (`docs/research/2026-08-23-thothii-metadata-publication-qdrant-seams.md:20-43`; `docs/research/2026-08-23-thothii-metadata-publication-qdrant-seams.md:239-301`). Anche il piano di migrazione rinvia il lifecycle/admin delle relationship logiche (`docs/plans/2026-08-26-metadata-catalog-from-thothai.md:681-691`). + +## Direzione confermata nel grill + +Il 31 agosto 2026 è stato concordato il seguente flusso minimo: + +1. Le foreign key dichiarate continuano a essere sincronizzate come Catalog Relationship fisiche. +2. Le foreign key ipotetiche sono salvate una sola volta come Logical Relationship fra colonna + sorgente e colonna destinazione; le tabelle sono ricavate dalle colonne e la UI le presenta nel + relativo contesto tabella. +3. Una relationship inferita è marcata `generated`; una relationship aggiunta dall'utente non lo è. +4. La cancellazione logica conserva il record e impedisce a una ricostruzione di riattivarlo. +5. La cancellazione fisica rimuove il record; una ricostruzione può ricrearlo se viene nuovamente + inferito. +6. La ricostruzione è additiva: conserva le relationship attive già presenti, non rimuove quelle + non più inferibili e non modifica le relationship manuali. +7. L'inferenza usa nomi, primary key e compatibilità dei tipi. Non campiona valori del DWH. +8. La relationship è l'unica fonte di verità: non viene duplicata in stringhe `fk_field` sulle + colonne. +9. I nomi vengono confrontati senza distinzione fra maiuscole/minuscole e normalizzando snake case, + kebab case e camel case. La regola riconosce anche casi come `user_id → users.id`, richiede tipi + compatibili e una sola destinazione possibile; riconosce inoltre un nome PK non generico con un + unico proprietario e la convenzione `*time_key → dim_time.`. Le colonne sorgenti + possono appartenere a PK composite, mentre nomi generici isolati come `id`, `key`, `code` e `pk` + non costituiscono evidenza. I casi ambigui vengono ignorati e non si usa fuzzy matching o un LLM. +10. La ricostruzione parte da un'azione amministrativa esplicita `Rebuild generated relationships`, + separata dalla sincronizzazione dello schema. +11. Le relationship cancellate logicamente restano consultabili tramite filtro e possono essere + riattivate con un'azione `Restore`. + +Non restano decisioni di dominio aperte per il flusso minimo. La progettazione UX e il seam tecnico +sono descritti nelle sezioni seguenti. + +## Lacuna UX emersa nel grill + +La vista Fleet `Relationships` esiste, ma nella UI di produzione non ha oggi un punto di ingresso +raggiungibile. La riga del database espone sincronizzazione, tabelle, dettagli, modifica e rimozione, +ma non le relationship; inoltre i tab `Overview / Tables / Relationships` appartengono soltanto alla +presentazione legacy. Il test di navigazione esistente esercita anch'esso la presentazione legacy, +non quella Fleet (`frontend/src/shell/database-management/DatabaseGrid.tsx:122-180`; +`frontend/src/shell/database-management/DatabaseForm.tsx:442-475`; +`frontend/src/shell/database-management/DatabaseForm.tsx:517-546`; +`frontend/src/shell/DatabaseManagementPage.test.tsx:2635-2703`). + +Se aperta programmaticamente, la vista mostra soltanto `Physical relationships` in sola lettura. La +toolbar contiene ricerca, conteggio, `Refresh`, un selettore azione con esecuzione esplicita e +`Sync history`; la griglia offre unicamente `Details`, che apre il drawer della relationship fisica. +Sono assenti ingresso visibile, aggiunta manuale, ricostruzione delle generated relationship, origine, +stato attivo/cancellato, cancellazione logica, cancellazione fisica e ripristino +(`frontend/src/shell/database-management/DatabaseRelationships.tsx:114-225`). + +I pattern Fleet già consolidati da riutilizzare sono: + +- una sola griglia nel livello corrente, con breadcrumb e controllo Back; +- azioni di pagina nel selettore `Choose an action…` con `Run action` e motivo visibile quando + indisponibili; +- azioni della singola riga nella colonna finale fissata a destra; +- form e dettagli in un drawer modeless che restituisce il focus al controllo di origine; +- conferme distruttive inline o nel drawer, non tramite una nuova pagina; +- toast per accettazione o errore e feedback persistente soltanto per le operazioni lunghe; +- card di errore con Retry ed empty state che indica la prossima azione possibile. + +ThothAI non offre un modello UX da copiare. L'inferenza è nascosta fra 21 bulk action della lista +database, non mostra avanzamento in tempo reale e restituisce soltanto messaggi a fine richiesta. Il +CRUD manuale vive in un'altra schermata Django Admin, non distingue origine o stato, offre soltanto +la cancellazione fisica e il form di aggiunta ha una validazione server strutturalmente incoerente +con le select popolate dal browser +(`Thoth/ThothAI/backend/thoth_core/admin_models/admin_sqldb.py:252-274`; +`Thoth/ThothAI/backend/thoth_core/admin_models/admin_relationship.py:58-119`; +`Thoth/ThothAI/backend/thoth_core/admin_models/admin_relationship.py:218-313`). + +## UX confermata + +Il 31 agosto 2026 sono state confermate le seguenti scelte: + +1. La colonna Actions della riga database espone un accesso diretto `Relationships`, accanto a + `Tables`. La vista conserva breadcrumb e controllo `Back to databases` esistenti. +2. La pagina presenta una sola griglia `Relationship map`, contenente relationship `Physical`, + `Generated` e `Manual`. Le colonne sono Source table, Source column, Target table, Target column, + Origin, Status e Actions. Constraint e regole update/delete rimangono nel drawer delle FK fisiche. +3. Un filtro visibile seleziona `Active`, `Excluded` o `All`; il valore predefinito è `Active`. Le FK + fisiche sono consultabili ma non modificabili da questa vista. +4. `Add relationship` è il pulsante primario visibile nella toolbar. Apre il drawer standard con i + quattro campi Source table, Source column, Target table e Target column e i comandi `Cancel` e + `Add relationship`. Il flusso minimo gestisce una sola coppia di colonne e mostra la validazione + accanto al campo interessato. +5. `Rebuild generated relationships` entra nell'attuale selettore `Choose an action…`, insieme a + `Synchronize physical relationships` e `Synchronize full schema`, con esecuzione esplicita tramite + `Run action`. `Refresh` ricarica la griglia; `Sync history` resta riservato alla sincronizzazione + fisica. +6. Il drawer di dettaglio contiene le azioni sulle relationship logiche. `Exclude` realizza la + cancellazione logica, `Delete permanently` quella fisica e `Restore` riattiva una relationship + esclusa. La conferma avviene nel drawer e spiega rispettivamente che la ricostruzione non + riattiverà un record escluso e potrà invece ricreare un record eliminato definitivamente. +7. La ricostruzione non introduce un nuovo sistema di job o di storico. Durante l'esecuzione mostra + `Rebuilding…`; al termine un toast riporta added, already present, excluded e ambiguous. Add, + Exclude, Delete permanently e Restore producono toast specifici; gli errori mantengono aperto il + contesto corrente. L'empty state propone di ricostruire dai nomi o aggiungere manualmente. +8. L'inferenza non usa AI. È codice deterministico nel backend del catalogo, basato su normalizzazione + dei nomi, primary key/unicità, compatibilità dei tipi e assenza di ambiguità. Non usa LLM, + embedding o campionamento dei dati. L'AI consuma la mappa risultante per comprendere lo schema, + ma non la costruisce. + +## Seam tecnico confermato + +Il Catalog PostgreSQL è l'unica fonte modificabile delle Logical Relationship. Le relationship +fisiche e logiche restano in modelli distinti, coerentemente con ADR-0006, mentre un servizio profondo +espone a API e UI una sola mappa discriminata per origine e stato. + +All'avvio o alla ripresa di una sessione, il backend materializza le sole relationship attive in una +snapshot JSON canonica e immutabile, collegata alla stessa lease della configurazione runtime. La +snapshot comprende anche le FK fisiche e conserva l'ordine delle coppie composite. Se due record hanno +gli stessi endpoint, la FK fisica ha precedenza. + +Quando la snapshot è presente, l'harness la usa come fonte esclusiva delle relationship e continua a +leggere dalle annotazioni Git-pinned soltanto descrizioni, sinonimi, concetti e altri metadati. Non +scrive `annotations.yaml` e non interroga direttamente il Catalog. Una snapshot dichiarata ma assente, +invalida o incoerente con lo schema fisico fallisce esplicitamente; un runtime legacy che non dichiara +la snapshot mantiene il precedente comportamento di compatibilità. + +Questa proiezione non è un secondo store: non può essere modificata, viene eliminata insieme alla +configurazione runtime e una sessione Pi vede una mappa stabile per tutta la propria vita. La decisione +duratura è registrata in ADR-0012. + +La proiezione e la ricostruzione sono ammesse soltanto dopo una sincronizzazione completa della +versione corrente del database e vengono serializzate con le mutazioni del catalogo. Un catalogo mai +sincronizzato, reso stale da una modifica della configurazione o invalidato da metadata cleanup non +può quindi diventare accidentalmente la fonte esclusiva del runtime. Il cleanup esplicito di una +tabella o colonna endpoint è anche il confine distruttivo del tombstone: rimuove definitivamente la +relationship esclusa, perché conservarla richiederebbe una seconda identità testuale denormalizzata. diff --git a/docs/testing/2026-08-31-metadata-privacy-description-test-plan.md b/docs/testing/2026-08-31-metadata-privacy-description-test-plan.md new file mode 100644 index 00000000..75d7bc08 --- /dev/null +++ b/docs/testing/2026-08-31-metadata-privacy-description-test-plan.md @@ -0,0 +1,428 @@ +# Piano di test: metadati di schema, campi sensibili e descrizioni AI + +- Data: 2026-08-31 +- Stato: proposto +- Baseline analizzata: `f586152` sul branch `test/database-baseline` + +## 1. Obiettivo + +Validare insieme le tre capacità recentemente introdotte in Database Management: + +1. acquisizione autorevole dei metadati fisici di uno schema esterno; +2. proposta e revisione umana dei campi sensibili dal punto di vista privacy; +3. generazione, revisione e consolidamento delle descrizioni di tabelle e colonne. + +Il piano è risk-based: perdita o corruzione di metadati, lettura o invio di valori protetti e +applicazione di una selezione al target sbagliato sono rischi bloccanti. La qualità linguistica dei +testi AI è invece valutata separatamente dal contratto tecnico, perché l'output del modello è una +proposta soggetta a revisione umana. + +Questo documento non costituisce una certificazione normativa o GDPR: verifica i controlli tecnici +e il flusso operativo implementati dal prodotto. + +### 1.1 Assunzione operativa: ambiente completamente sacrificabile + +Per indicazione esplicita del proprietario, l'installazione sul Mac è esclusivamente di sviluppo e +test. Non contiene produzione e non esiste alcun requisito di conservazione dello stato locale. + +- `catalog-db`, migrazioni, configurazioni locali, run, eventi, flag, descrizioni curate e generate + possono essere cancellati e ricreati tutte le volte necessarie; +- il database reale già collegato è la fonte autorevole dalla quale ricostruire il catalogo ed è la + sorgente primaria per validare schema, classificazione privacy e generazione delle descrizioni; +- reset completi, failure injection, dati incoerenti deliberati e prove distruttive sul catalogo + locale sono ammessi senza backup o procedura di rollback dell'ambiente; +- compatibilità con stato locale preesistente, sessioni legacy e vecchie revisioni del catalogo non + è un gate di questo piano; +- le prove di atomicità, idempotenza e preservazione dei campi restano necessarie perché verificano + il comportamento del prodotto dentro un ciclo di test, non perché debbano proteggere il Mac. + +Il database sorgente resta normalmente read-only: quando una prova richiede DDL o mutazioni fra +scan e conferma si usa uno schema di test esplicitamente scrivibile o la fixture supplementare. La +disponibilità a buttare lo stato locale non elimina il rischio di inviare dati personali a un +provider esterno: proprio questa non-esfiltrazione è uno degli esiti principali da collaudare. + +## 2. Terminologia e risultato atteso + +I tre campi testuali del catalogo hanno autorità diverse e non devono essere confusi: + +| Campo | Origine e autorità | Comportamento atteso | +| --- | --- | --- | +| `sourceComment` | Commento fisico acquisito dalla sorgente | Si aggiorna con la sincronizzazione e non è modificabile come contenuto curato. | +| `generatedDescription` | Proposta prodotta dall'AI o corretta nel catalogo | È salvata separatamente, può essere rigenerata esplicitamente e non sovrascrive gli altri due campi. | +| `description` | Descrizione curata dall'amministratore | Cambia solo tramite modifica esplicita o consolidamento di una proposta selezionata. | + +Nel seguito, “commento generato” indica `generatedDescription`. La generazione non deve mai +modificare `sourceComment`; il passaggio a `description` avviene solo con il consolidamento umano. + +## 3. Riferimenti e baseline + +Il comportamento da verificare deriva da: + +- stato corrente del progetto in `PROJECT_STATE.md`; +- [contratto Catalog Schema Snapshot](../contracts/catalog-schema-snapshot.md); +- [piano del Metadata Catalog](../plans/2026-08-26-metadata-catalog-from-thothai.md); +- [specifica della generazione descrizioni](../plans/2026-08-28-ai-catalog-description-generation-spec.md); +- [ADR 0007: sincronizzazione autorevole durevole](../adr/0007-durable-authoritative-schema-synchronization.md); +- [ADR 0009: un solo run sequenziale di generazione](../adr/0009-use-one-sequential-description-generation-run.md); +- [ADR 0010: campioni reali limitati](../adr/0010-allow-bounded-real-source-samples-for-description-generation.md); +- [ADR 0011: Sensitive Data Flag](../adr/0011-gate-source-samples-with-a-sensitive-data-flag.md); +- [accettazione AI del 2026-08-29](./2026-08-29-ai-catalog-description-generation-acceptance.md). + +L'accettazione del 2026-08-29 è una baseline utile ma non chiude il gate attuale: precede i commit +`0736983` e `cb40c09`, inviava campioni reali inventati e documentava che un valore campione era +stato ripreso nella descrizione. Deve quindi essere ripetuta sul comportamento privacy corrente. + +## 4. Perimetro + +### Incluso + +- trasporti `postgres_direct`, `rest_api` e `ssh_tunnel`; +- RPC REST tipizzato `POST /rpc/schema_snapshot` e fallback singolo read-only su + `POST /rpc/run_query`; +- metadati di tabelle, colonne, commenti sorgente, tipi, default, nullabilità, posizioni PK e coppie + FK ordinate; +- scope di sincronizzazione `tables`, `columns`, `relationships` e `all`; +- run durevoli, conferma delle differenze distruttive, cancellazione, recovery, eventi SSE e + fallback polling; +- Sensitive Data Flag, suggerimenti AI strutturali, review draft e storico operativo; +- scope di generazione `selected_columns`, `selected_tables`, `all` e `missing`; +- campionamento read-only, valori sintetici per colonne protette, batching, retry, stop, Unlock, + storico e consolidamento; +- uso controllato del database reale già collegato, inclusi schema e valori non sensibili, per il + collaudo end-to-end con fake provider e provider configurati; +- permessi, minimizzazione dei dati, redazione di log/errori e resistenza a input ostili. + +### Escluso o rinviato + +- applicazione del Sensitive Data Flag allo schema-linking/LSH del core, esplicitamente rinviata; +- sostituzione di `annotations.yaml`, pubblicazione Qdrant e cutover del runtime NL→SQL; +- alias, sinonimi, concetti, value descriptions e relazioni logiche; +- audit delle decisioni umane sul flag: per disegno si conserva solo il booleano corrente; +- DDL o scritture sul database sorgente; +- conservazione di cataloghi, run, descrizioni o sessioni locali precedenti al reset; +- compatibilità all'indietro con formati o migrazioni legacy non appartenenti alla baseline corrente. + +## 5. Priorità e strategia + +| Priorità | Significato | Esempi | +| --- | --- | --- | +| P0 | Gate bloccante di sicurezza o integrità | Nessuna lettura/invio di valori protetti; applicazione atomica; scope esatto; segreti non esposti. | +| P1 | Contratto funzionale necessario al rilascio | Trasporti, stati dei run, recovery, batching, review e consolidamento. | +| P2 | Qualità, UX e robustezza non distruttiva | Copy, focus, benchmark del modello, carico e degrado SSE. | + +Le prove sono distribuite su quattro livelli: + +1. **Contratto/unità**: repository in memoria, finti connector e Model Completer; nessuna rete. +2. **Integrazione**: catalogo PostgreSQL ricreabile via Docker, database reale come sorgente, + fixture supplementare, fake REST/SSH e Model Completer osservabile. +3. **UI/E2E locale**: component test con MSW e almeno un flusso Playwright sullo stack locale. +4. **Accettazione L2**: provider configurato realmente e database reale dopo review dei flag. + +I test deterministici verificano il contratto. Le prove con un modello reale verificano +compatibilità e qualità, ma non sostituiscono i gate P0. + +## 6. Ambiente e dati di test + +### 6.1 Ambiente minimo + +- stack locale con `catalog-db` eliminabile, migrazioni ripetibili e backend/frontend della stessa + baseline; +- database reale già collegato come sorgente primaria, con utenza capace di `SELECT` ma non di + `INSERT`, `UPDATE`, `DELETE`, DDL o cambio di schema; +- PostgreSQL effimero supplementare solo per le mutazioni controllate che non devono essere fatte + sul database reale; +- server REST fake in grado di servire snapshot valido, capability `unavailable`, 404, risposta + parziale/malformata e fallback `run_query`; +- server OpenSSH effimero con `known_hosts` esatto e varianti host key errata/assente; +- Model Completer fake che registra transitoriamente i messaggi e restituisce esiti programmabili; +- browser senza segreti in Web Storage e raccolta di log backend/SSE/API per le scansioni canary. + +Le prove PostgreSQL di integrazione non devono risultare `skip`: la disponibilità di Docker è una +precondizione del gate. Il catalogo locale può essere azzerato prima di ogni wave senza snapshot o +backup del suo stato precedente. + +### 6.2 Database reale e fixture supplementare `catalog_qa` + +La prima baseline viene acquisita dal database reale collegato. Il test registra soltanto inventario +strutturale, conteggi e risultati sanitizzati; poi svuota il catalogo locale e dimostra di poterlo +ricostruire dalla stessa sorgente. Non è richiesto preservare alcun metadato locale precedente. + +La fixture `catalog_qa` integra il database reale soltanto quando servono casi controllabili o +mutazioni che la sorgente reale non contiene. Deve includere almeno: + +- `customers`: UUID PK, nome, email, codice fiscale, telefono, data di nascita, indirizzo e note; +- `orders`: FK verso `customers`, importo numerico, stato, timestamp, default e campi nullable; +- `order_lines`: PK composta e relazione composta ordinata; +- `clinical_events`: campi sanitari evidenti e tabella partizionata; +- `products`: SKU pubblico, categoria, prezzo e flag booleano; +- `empty_table`: nessuna riga ma struttura valida; +- `wide_entity`: almeno 23 colonne e metadati lunghi, per forzare batch `10 + 10 + 3` e il limite + dimensionale del messaggio; +- identificatori quotati, commenti Unicode/italiani, commenti null e oggetti fuori dallo schema. + +Usare valori canary inventati e univoci, per esempio: + +- `PRIV_EMAIL_CANARY_...`, `PRIV_TAX_CANARY_...`, `PRIV_HEALTH_CANARY_...` nelle colonne protette; +- `PUBLIC_SKU_CANARY_...` in una colonna esplicitamente non sensibile; +- `SECRET_API_CANARY_...` solo nel secret store del test. + +I canary protetti non devono comparire nei messaggi al modello, nel catalogo, negli eventi, nelle +API, nel DOM o nei log. Il canary pubblico può apparire nel messaggio al provider entro i limiti +documentati e dopo la disclosure esplicita dell'utente. Sul database reale la stessa proprietà va +provata soprattutto osservando la proiezione SQL e il payload transitorio: i valori protetti non +devono essere letti, e nessun valore grezzo deve entrare nell'evidenza conservata. + +### 6.3 Mutazioni della sorgente + +Preparare tre revisioni dello schema: + +- **A — iniziale**: struttura completa e commenti sorgente valorizzati; +- **B — distruttiva**: rimozione di una tabella, una colonna e una FK, più aggiunta di una nuova + colonna sensibile per nome; +- **C — race di conferma**: modifica ulteriore fra piano distruttivo e conferma, per provare il + re-scan. + +Queste revisioni possono vivere nella fixture o in uno schema reale esplicitamente dichiarato +scrivibile. Fra una prova e l'altra è consentito eliminare completamente il catalogo locale, +riapplicare le migrazioni e ripartire dal database reale. + +## 7. Casi di test — acquisizione dei metadati + +| ID | P | Livello | Scenario | Risultato atteso | +| --- | --- | --- | --- | --- | +| MET-01 | P0 | API/Integrazione | Avvio senza binding raggiungibile, con versione database obsoleta o senza `database.manage`. | Il run non parte; risposta sicura e catalogo invariato. I segreti restano write-only. | +| MET-02 | P0 | Integrazione | Reset completo del catalogo e `all` sul database reale collegato, senza mock del client `pg_catalog`; ripetizione sulla fixture A per gli edge case assenti. | Il catalogo viene ricostruito da zero con snapshot esatta di tabelle, colonne, `sourceComment`, tipo, default, nullabilità, PK e FK ordinate; stato `succeeded`; nessuna scrittura alla sorgente. | +| MET-03 | P1 | Contratto/Integrazione | Stessa fixture via RPC REST tipizzato. | `schemaVersion: 1` e capability sono validate strettamente; risultato normalizzato uguale a MET-02. `unavailable` non è interpretato come collezione vuota. | +| MET-04 | P0 | Contratto/Integrazione | `/schema_snapshot` assente, fallback `run_query`; poi fallback assente, parziale, non JSON o con campi extra/mancanti. | Il fallback usa una sola query read-only. Ogni errore o snapshot invalida fallisce senza modifiche parziali; nessun fallback nasconde un errore operativo diverso da capability assente. | +| MET-05 | P1 | Integrazione | Accesso `ssh_tunnel` con host key corretta, errata e assente; errore durante apertura/chiusura. | Parità con MET-02 nel caso valido; fail-closed negli altri casi; processi, lease e file-segreto sempre rilasciati. | +| MET-06 | P0 | API | Esecuzione separata di `tables`, `columns`, `relationships` e `all`, con e senza selezione tabelle. | Ogni scope è autorevole solo nel proprio confine; nessun record fuori scope cambia o viene eliminato. Selezioni duplicate/inesistenti sono rifiutate atomicamente. | +| MET-07 | P0 | API/Integrazione | Ripetizione idempotente della fixture A dopo modifica di `description`, `generatedDescription` e `sensitive`. | Nessun diff fisico spurio; contenuti curati, proposte AI e flag delle entità ancora presenti sono preservati. Una nuova colonna nasce con `sensitive=false`. | +| MET-08 | P0 | API/Integrazione | Passaggio A→B, conferma assente/errata/scaduta e passaggio A→B→C prima della conferma. | Stato `awaiting_confirmation`, piano visibile e nessuna applicazione anticipata. La conferma valida provoca re-scan; se il diff cambia viene emesso un nuovo piano/token e il vecchio non applica nulla. Apply atomica oppure zero modifiche. | +| MET-09 | P0 | API/Integrazione | Timeout, disconnessione, capability incompleta o eccezione durante scan/apply. | Stato terminale coerente, errore sanitizzato, catalogo precedente intatto e lock rilasciato. Nessun segreto, SQL sensibile o stack trace nelle API/eventi. | +| MET-10 | P1 | Worker | Cancel in `queued`, `running`, `awaiting_confirmation` e `applying`; retry, restart con run attivo e lease scaduto. | Cancel è efficace solo prima di apply ed è rifiutato durante apply; retry crea un nuovo run. Recovery non duplica l'apply, marca correttamente i run interrotti, rilascia il lock e rimuove snapshot/diff/token interni non più necessari. | +| MET-11 | P0 | API | Due operazioni sullo stesso database: sync, cleanup, connection test, edit o generazione; in parallelo, operazioni su database diversi. | Una sola operazione possiede il database; conflitto 409 sicuro sullo stesso target. Nessun lock cross-database non previsto e nessuna release del token altrui. | +| MET-12 | P1 | UI | Avvio dai menu database/tabella, visualizzazione piano, conferma, history drawer, chiusura drawer, perdita SSE e polling. | Scope e selezione inviati sono esatti; azioni non eleggibili restano visibili con motivo; chiudere il drawer non ferma il run; replay/polling deduplicano gli eventi e aggiornano griglie/KPI. | +| MET-13 | P1 | E2E | Cleanup manuale di tabelle/colonne/relazioni e successiva sincronizzazione. | Cleanup modifica solo il catalogo; la sorgente resta invariata; un sync autorevole ripristina gli oggetti ancora presenti in sorgente. | +| MET-14 | P0 | Integrazione | Cambio binding/versione fra scan e apply ed errore iniettato a metà transazione. | La freshness viene ricontrollata sotto lock; il run fallisce senza righe parziali e conserva integralmente il catalogo precedente. | +| MET-15 | P1 | API/Worker | Replay SSE con `Last-Event-ID`/`after`, polling concorrente e retention oltre 30 giorni. | Cursori monotoni e nessun duplicato; gli eventi scaduti vengono potati senza corrompere run e stato finale. | + +## 8. Casi di test — identificazione e protezione dei campi sensibili + +| ID | P | Livello | Scenario | Risultato atteso | +| --- | --- | --- | --- | --- | +| PRV-01 | P0 | Repository/API | Prima sincronizzazione, re-sync e aggiunta di una colonna. | Il default è `false`; il valore umano delle colonne esistenti è preservato; la nuova colonna è esplicitamente da riesaminare ma non riceve uno stato audit inventato. | +| PRV-02 | P0 | API | Suggerimento per un database, tabelle selezionate e colonne selezionate; database multipli, target duplicati o mancanti. | Il provider riceve esattamente le colonne dello scope. Input ambigui sono rifiutati prima della chiamata e nessun flag cambia. | +| PRV-03 | P0 | Contratto | Ispezione del messaggio al classifier. | Sono presenti solo database, schema, tabella, colonna, tipo, nullabilità, PK e FK. Non compaiono righe, valori, commenti, descrizioni, flag corrente o segreti. | +| PRV-04 | P1 | Contratto | `wide_entity`, identificatori lunghi e limite byte. | Ordine deterministico, batch massimi di dieci colonne e rispetto del limite messaggio; una singola colonna non rappresentabile fallisce prima del provider con errore sicuro. | +| PRV-05 | P0 | API | Risposta valida, fenced/prosa, JSON malformato, target mancante/duplicato/ignoto e provider failure. | Ogni colonna richiesta compare una sola volta. Una classificazione invalida viene ritentata una volta; dopo esaurimento si ottiene errore sanitizzato e nessuna modifica. | +| PRV-06 | P0 | UI/API | Apertura draft, modifica manuale, chiusura/reload e salvataggio. | La proposta non è persistita prima di Save; reload la scarta. Il reviewer può invertire scelte; si salvano solo colonne cambiate con versione ottimistica; un conflitto richiede reload. | +| PRV-07 | P1 | Repository/UI | Tentativi completati, falliti e attivi al restart. | Ogni tentativo ha un run distinto con scope, modello, contatori ed eventi sanitizzati; startup marca `interrupted` i run attivi. Storico newest-first senza target ID, proposte, prompt, output grezzo o diagnostica provider. | +| PRV-08 | P0 | Integrazione | Generazione descrizioni su target con canary protetti. | Le colonne protette sono assenti dalla proiezione SQL, non semplicemente filtrate dopo la lettura. Se non rimangono colonne leggibili non viene eseguita una `SELECT`. Nessun canary protetto esce dal processo. | +| PRV-09 | P0 | Contratto/Integrazione | Tabella mista con colonne sensibili e pubbliche. | Per le sensibili il prompt contiene valori plausibili, deterministici e limitati derivati dai soli metadati, nello stesso formato dei campioni e senza etichettarli al modello come sintetici. Per le pubbliche: massimo cinque righe e cinque valori rappresentativi, valori troncati e transazione read-only chiusa con rollback. | +| PRV-10 | P1 | API | Cambio `false→true→false` dopo una descrizione già generata. | Il testo esistente non viene rigenerato retroattivamente. Solo le generazioni future cambiano fonte del contesto; tornando `false` il campionamento reale torna eleggibile. | +| PRV-11 | P0 | API/UI | Utente senza `database.manage`, modello non configurato, catalogo/provider indisponibile e richiesta interrotta. | Controlli nascosti/disabilitati in UI e rifiuto server-side; errori non espongono dettagli. Un tentativo fallito compare nello storico senza trasformarsi in audit della decisione umana. | +| PRV-12 | P1 | L2 | Corpus strutturale etichettato con identificatori personali, credenziali/token, salute, finanza, localizzazione e controlli non sensibili/ambigui, in inglese e italiano. | Si misurano precisione, recall e falsi negativi per modello. I campi critici mancati sono sottoposti al product owner; la soglia quantitativa va ratificata prima di diventare gate, perché il classifier è advisory e human-in-the-loop. | +| PRV-13 | P0 | UI/E2E | Modifica di un flag nella review senza Save e tentativo immediato di generare descrizioni. | Gate di rilascio da formalizzare: la generazione deve essere bloccata finché il draft non è salvato o scartato. In alternativa la UI deve dichiarare inequivocabilmente che verrà usato il valore persistito; non è accettabile mostrare “protetto” e campionare come non protetto. | + +## 9. Casi di test — generazione e consolidamento dei commenti + +| ID | P | Livello | Scenario | Risultato atteso | +| --- | --- | --- | --- | --- | +| GEN-01 | P0 | Config/API | Modelli validi, default, endpoint anonimo esplicito, secret ref mancante/errato e nessun modello. | Al browser arrivano solo ID, label e default. Nessuna chiave, provider payload o configurazione privata è esposta; feature disabilitata in modo comprensibile se non configurata. | +| GEN-02 | P0 | API | `selected_columns`, `selected_tables`, `all`, `missing`, target duplicati/inesistenti e zero eleggibili. | Scope esatto; `missing` include null/vuoto/whitespace e salta proposte esistenti; `all` sostituisce solo dopo conferma esplicita; input invalido non crea run. | +| GEN-03 | P1 | Worker | 23 colonne più tabelle, metadati e campioni lunghi. | Colonne prima delle tabelle per lo scope globale; richieste omogenee, sequenziali, massimo dieci target e sotto i limiti byte; contesto tabella aggiornato dopo le colonne. | +| GEN-04 | P0 | Contratto | JSON puro, un solo code fence JSON completo, prosa extra, payload multipli, target mancante/duplicato/ignoto, descrizione vuota o oltre limite. | Sono accettati solo i primi due formati validi. Una mappatura ambigua non applica alcun risultato del batch e conta come errore tecnico. | +| GEN-05 | P1 | API | Esito `generated` e `non_generatable` in workspace italiano/inglese. | Testo generato trimmato e salvato; il testo standard non generabile è localizzato dall'applicazione, non copiato dal provider. Gli errori tecnici non scrivono tale testo. | +| GEN-06 | P0 | Worker | Errore transiente, errore esaurito isolato, tre batch falliti consecutivi e successo fra due errori. | Helper con al massimo un retry e nessun fallback modello. Errori isolati portano a `completed_with_errors`; tre consecutivi a `failed`; un successo azzera il contatore. | +| GEN-07 | P0 | Integrazione | Successi seguiti da cancel, interruption o failure. | Ogni risultato valido è persistito subito e resta disponibile; il target fallito non riceve dati ambigui. “Generate Missing” consente il recupero naturale senza resume automatico. | +| GEN-08 | P0 | API | Secondo run durante un run attivo e modifica/sync/cleanup/consolidamento sul database posseduto. | Un solo run di generazione attivo nell'installazione; 409 chiaro al secondo Start. Il database target resta riservato e le operazioni incompatibili non alterano selezione o dati. | +| GEN-09 | P0 | Integrazione/Worker/UI | Stop in queued/running con helper attivo e con una `SELECT` sorgente deliberatamente bloccata/lenta. | Helper e query/connessione vengono terminati entro 5 secondi, stato `cancelled`, nessuna chiamata modello dopo Stop, risultati precedenti conservati ed eventi consultabili. Un test con sampler fake non è sufficiente. | +| GEN-10 | P0 | Worker/API | Restart con run `queued/running`; Unlock con worker/helper vivo e con run realmente stale; race Unlock/Start. | Startup marca `interrupted` senza replay. Unlock è rifiutato se esiste lavoro locale vivo e non può liberare la reservation di un nuovo run. | +| GEN-11 | P1 | Integrazione | Sorgente campioni indisponibile ma metadati validi. | La generazione prosegue metadata-only con un solo warning sicuro; nessun tentativo alternativo espone dettagli di connessione. | +| GEN-12 | P1 | API/UI | SSE disconnesso, replay da sequence, polling concorrente e riapertura storico. | Eventi persistiti, ordinati e deduplicati; history newest-first; contatori/stato finali coerenti; nessun prompt, campione, risposta completa o stack trace. | +| GEN-13 | P0 | UI/API | Revisione manuale della proposta e consolidamento selettivo di tabelle/colonne, inclusi target vuoti/stale. | Si copia solo `generatedDescription` non vuota dei target risolti; conteggio `copied/skipped` corretto; operazione atomica; `sourceComment` invariato. | +| GEN-14 | P0 | UI/Sicurezza | Tutti gli scope di generazione da database, tabelle e colonne selezionate, utente senza permesso, metadata contenente istruzioni ostili. | Prima di ogni Start, inclusi `selected_tables` e `selected_columns`, compare la disclosure “fino a cinque righe e cinque valori”. Il server applica comunque l'autorizzazione. Metadati e valori sono trattati come dati non fidati e l'output resta nel contratto JSON. | +| GEN-15 | P1 | L2 | Run reale sul modello di default e almeno un endpoint alternativo supportato, usando il database reale dopo la review dei flag e un role read-only. | Descrizioni nella lingua workspace, coerenti con struttura/commenti e senza fatti inventati critici; almeno una colonna e una tabella consolidate. Nessun valore protetto o segreto compare in request osservabile, eventi, API, persistenza o log. | +| GEN-16 | P1 | Integrazione | Fastify→worker→processo Python→LiteLLM→endpoint OpenAI-compatible locale di cattura. | Routing provider/model, header API key, `disableThinking`, singolo retry, timeout, limite output e payload sono corretti; chiave e diagnostica non risalgono a stdout, API o log applicativi. | + +## 10. Percorso E2E prioritario + +Il caso `E2E-01` deve attraversare le tre feature senza sostituire i test di contratto: + +1. azzerare `catalog-db`, riapplicare le migrazioni e verificare che il role del database reale sia + realmente read-only; +2. eseguire `all` sul database reale e confrontare catalogo e snapshot sorgente; usare la fixture A + in una seconda esecuzione per gli edge case mancanti; +3. richiedere suggerimenti privacy sull'intero database; +4. modificare almeno una proposta e salvare i flag revisionati; +5. avviare `missing` dopo la disclosure, usando un Model Completer osservabile; +6. dimostrare che i canary protetti non sono letti né inviati e che il canary pubblico rispetta i + limiti; +7. correggere una `generatedDescription` e consolidare una tabella e una colonna; +8. applicare la fixture B, controllare il piano distruttivo e introdurre C prima della conferma; +9. confermare dopo il re-scan e verificare atomicità, preservazione di descrizioni/flag delle entità + superstiti e `sensitive=false` sulla nuova colonna; +10. perdere la connessione SSE, riaprire entrambi gli storici e verificare replay, polling e KPI. + +Il percorso deve essere eseguito con fake provider dopo ogni reset rilevante e, in forma ridotta, +con il provider configurato sul database reale dopo che i flag sono stati revisionati e salvati. +Non è richiesto ripristinare lo stato locale precedente al test. + +## 11. Valutazione qualitativa dei modelli + +La qualità non deve confondersi con la sicurezza: anche un classifier perfetto non autorizza +l'esfiltrazione di un canary protetto e una descrizione elegante non rende valido un payload +ambiguo. + +### 11.1 Classificazione privacy + +Per ogni modello registrare matrice di confusione, precisione, recall e falsi negativi per categoria. +Eseguire almeno tre iterazioni sul corpus fisso per rilevare instabilità. Fino alla ratifica di una +soglia da parte del product owner, il risultato è un gate di review: ogni falso negativo su +credenziali/token, identificatori fiscali, dati sanitari o finanziari richiede accettazione esplicita +o correzione prima del rilascio operativo. + +### 11.2 Descrizioni generate + +Valutare ogni testo da 0 a 2 su: + +- correttezza rispetto a struttura e commenti sorgente; +- specificità e utilità per un revisore; +- lingua e chiarezza; +- assenza di istruzioni seguite dai dati non fidati o fatti inventati; +- assenza di valori protetti e segreti. + +Soglia proposta da ratificare: almeno 8/10, nessun punteggio 0 su correttezza o sicurezza. Un valore +esplicitamente non sensibile, reale o inventato, può essere ripreso entro il perimetro dichiarato; +un valore protetto non può mai esserlo. + +## 12. Copertura esistente e gap da chiudere + +| Area | Evidenza automatica già presente | Gap principale | +| --- | --- | --- | +| Snapshot e sincronizzazione | `backend/test/catalog-schema-introspector.test.ts`, `catalog-schema-routes.test.ts`, `catalog-table-introspector.test.ts`, `catalog-repository.integration.test.ts` | Introspezione `pg_catalog` realmente end-to-end, parità live dei tre trasporti e un unico E2E con re-scan distruttivo. | +| Privacy | `catalog-description-generation-routes.test.ts`, `catalog-description-generation-worker.test.ts`, `catalog-description-source-sampler.test.ts`, `catalog-synthetic-sample-value.test.ts` | Prova canary integrata query→prompt→API/log e benchmark reale post-ADR 0011. | +| Generazione | `catalog-description-generation-routes.test.ts`, `catalog-description-generation-worker.test.ts`, `catalog-description-generation.integration.test.ts` e test del helper | Accettazione reale aggiornata, cancellazione di una query PostgreSQL bloccata e integrazione ermetica fino all'endpoint LiteLLM locale. | +| UI | `DatabaseManagementPage.test.tsx`, `DescriptionGenerationDrawer.test.tsx`, `SensitiveDataSuggestionHistoryDrawer.test.tsx` | L'E2E Playwright corrente verifica soprattutto il layout, non il workflow funzionale. | + +Nuovi asset consigliati: + +- `backend/test/catalog-metadata-privacy-workflow.integration.test.ts`; +- `backend/test/fixtures/catalog-privacy-schema.sql` e snapshot REST equivalenti; +- integrazione a due PostgreSQL per le query reali `pg_catalog` e lo stop del sampler; +- endpoint OpenAI-compatible locale di cattura per GEN-16; +- `frontend/e2e/database-management-workflow.spec.ts`; +- corpus strutturale versionato per PRV-12, senza valori business; +- nuovo report di accettazione L2 che sostituisca il gate privacy del 2026-08-29. + +## 13. Ordine di esecuzione + +### Wave 1 — contratto rapido + +- parser snapshot, introspector, scope e diff; +- classifier strutturale, batching e validazione output; +- sampler, valori sintetici, prompt bounds e parser descrizioni; +- autorizzazione, redazione e race del coordinator. + +### Wave 2 — integrazione PostgreSQL + +- reset totale di `catalog-db`, bootstrap delle migrazioni e ricostruzione dal database reale; +- migrazioni e vincoli del repository; +- atomicità sincronizzazione/consolidamento; +- run ed eventi persistiti, restart e cancellation; +- `E2E-01` con fake provider e canary. + +### Wave 3 — frontend e stack locale + +- component test MSW; +- Playwright funzionale, perdita SSE e polling; +- verifica disclosure, review draft, history e consolidamento. + +### Wave 4 — accettazione L2 + +- provider reale sul database collegato dopo classificazione e review dei flag; +- benchmark PRV-12 e rubric GEN-15; +- scansione finale di canary e segreti; +- approvazione del product owner. + +Comandi di regressione: + +```bash +cd backend && npx vitest run +cd backend && npx tsc --noEmit -p . +cd backend && npm run build +cd frontend && npx vitest run +cd frontend && npx tsc -b +cd frontend && npm run build +cd frontend && npm run e2e +./scripts/build-docs.sh +``` + +Il report deve evidenziare esplicitamente eventuali test Docker/L2 saltati; un `skip` non equivale a +PASS del relativo gate. + +## 14. Evidenze da conservare + +Per ogni esecuzione registrare: + +- commit, configurazione pubblica dei modelli, versione fixture e trasporto; +- ID e stato finale dei run, contatori ed eventi sanitizzati; +- snapshot catalogo prima/dopo e piano distruttivo con metadati e conteggi sanitizzati, senza valori + grezzi delle righe sorgente; +- report test/JUnit, screenshot dei gate UI e risultato della scansione canary; +- matrice di confusione privacy e rubric delle descrizioni per le prove L2; +- difetti con ID del caso, severità, riproducibilità e decisione finale. + +Non allegare prompt completi, righe campione, output grezzi del provider, chiavi, digest o frammenti +di segreti. Il Model Completer spy deve verificare in memoria le asserzioni e scartare il payload al +termine del test. + +Non serve conservare backup del catalogo locale, run precedenti o descrizioni generate durante una +wave: l'evidenza è il report sanitizzato e la capacità di ricostruire nuovamente il risultato dalla +sorgente reale. + +## 15. Criteri di ingresso e uscita + +### Ingresso + +- baseline unica per backend/frontend e procedura verificata per eliminare e ricreare `catalog-db`; +- accesso al database reale collegato e inventario delle tabelle/colonne da includere nella review; +- fixture e canary supplementari approvati per i soli edge case controllati; +- role sorgente read-only verificato con una scrittura deliberatamente negata; +- fake connector/provider disponibili e log collection attiva; +- per L2, secret reference configurato senza materializzare il valore nell'evidenza. + +### Uscita + +- 100% dei casi P0 e P1 applicabili superati; nessun difetto Sev-1/Sev-2 aperto; +- almeno due ricostruzioni complete e coerenti del catalogo a partire dal database reale dopo reset + indipendenti; +- zero comparsa dei canary protetti e dei segreti fuori dalla sorgente/secret store del test; +- nessuna modifica parziale dopo errori, cancel o conferme stale; +- parità normalizzata dei trasporti supportati e nessuna integration PostgreSQL richiesta saltata; +- stati, contatori, eventi e history coerenti dopo success, partial failure, stop e restart; +- `sourceComment`, `generatedDescription` e `description` mantengono l'autorità prevista; +- accettazione L2 sul database collegato approvata dal product owner e + build/test/typecheck/documentazione verdi; +- ogni deviazione P2 o soglia qualitativa non ancora ratificata è documentata con owner e data. + +## 16. Rischi residui da rendere espliciti + +- `sensitive=false` è il default e il prodotto non conserva uno stato “review completata”: dopo ogni + reset, classificazione strutturale e salvataggio umano dei flag devono precedere qualunque run con + provider reale. Il catalogo è ricostruibile; un invio errato a un provider non lo è. +- Il flag protegge i valori campionati. Nomi, `sourceComment` e descrizioni sono metadati inviabili + al modello e possono contenere testo libero: se nel database reale includono PII serve una + decisione aggiuntiva di redazione, non una diversa aspettativa di test. +- La UI deve risolvere il caso di un flag modificato ma non salvato prima della generazione e deve + mostrare la disclosure anche per tabelle/colonne selezionate; il piano considera entrambi P0. +- L'AbortSignal corrente va provato contro una query PostgreSQL realmente bloccata: la sola + cancellazione del helper non dimostra che la lettura sorgente sia interrompibile. +- Lo storico dei Sensitive Data Suggestion Run è operativo, non un audit delle decisioni umane. +- La policy privacy non è ancora applicata allo schema-linking/LSH; nessun risultato di questo piano + deve essere presentato come copertura di quel percorso. +- Un provider reale resta non deterministico: il rilascio deve dipendere dai gate tecnici e dalla + review umana, non dalla ripetizione byte-identica delle descrizioni. +- L'esclusione fra operazioni e Unlock è in parte locale al processo. Se il deployment ammetterà più + repliche backend, servirà un gate aggiuntivo con due istanze contro lo stesso catalogo; non va + dedotta sicurezza multi-replica dai test single-process. diff --git a/docs/testing/evidence-lifecycle-test-plan.md b/docs/testing/evidence-lifecycle-test-plan.md new file mode 100644 index 00000000..dcbc5e00 --- /dev/null +++ b/docs/testing/evidence-lifecycle-test-plan.md @@ -0,0 +1,1030 @@ +# Piano di test E2E del ciclo di vita delle Evidence + +- Data di preparazione: 2026-08-31 +- Workspace di riferimento: `psd-clinical` +- Workspace di collaudo proposto: `psd-evidence-lab` + +## 1. Obiettivo + +Questo piano verifica, su servizi reali, l'intero ciclo di vita delle Evidence: + +1. creazione e registrazione di un workspace parallelo a PSD; +2. inserimento di Source Evidence e generazione delle Curated Evidence strutturate; +3. revisione, validazione, commit, acquisizione commit-addressed e inventario; +4. indicizzazione dense + BM25 in Qdrant; +5. ricerca tipizzata e uso nei processi core F1-F8; +6. inserimento incrementale, modifica e riacquisizione; +7. rename, relink, rimozione, orphan e retirement; +8. idempotenza, retention, concorrenza, recovery e fallimenti parziali; +9. isolamento, sicurezza, audit e non regressione di Schema, Memory e solved questions. + +Il test è un'accettazione manuale E2E assistita, non un sostituto delle suite automatiche. Deve +attraversare davvero Git, Workspace Management, host CLI, backend, core, Ollama, Qdrant e una +sessione Pi completa. + +## 2. Chiarimento essenziale: cosa significa CRUD Evidence oggi + +Non esiste un CRUD Evidence nella UI o in una API pubblica. Il repository Git del workspace è la +fonte autorevole; Qdrant è una proiezione ricostruibile. + +| Operazione funzionale | Gesto reale dell'utente | +| --- | --- | +| Inserimento | aggiungere un file in `evidence/source/`, eseguire `evidence prepare`, revisionare, validare, committare e pubblicare la revisione | +| Lettura | leggere source/curated in Git; cercare la Published Evidence via retrieval; verificare provenienza e citation | +| Modifica | modificare solo la Source Evidence, rieseguire prepare/review/validate e pubblicare una nuova revisione/generazione | +| Cancellazione | rimuovere la source, osservare l'orphan bloccante, poi decidere esplicitamente `--retire` oppure `--source` per il relink | +| Inventario | correlare manifest Git, materializzazione del commit, manifest ACTIVE, document generations, payload Qdrant e receipt di sessione | + +ThothII non modifica, committa o pusha il repository dell'autore. Il curatore non deve modificare +manualmente `manifest.yaml`, hash, generation, metadata invisibili o marker `tht:`. + +## 3. Fonti di verità e oracoli + +| Livello | Autorità o proiezione | Oracle da conservare | +| --- | --- | --- | +| Authoring Git | autorità | commit SHA, `source/`, `curated/`, `manifest.yaml`, `evaluation.yaml` | +| Registry snapshot | copia immutabile del commit attivo | workspace revision, `snapshot.json`, `evidence.manifest.json`, hash e conteggi file | +| Corpus Evidence | generazione pubblicata | puntatore `ACTIVE`, generation manifest, `document_generations` | +| Qdrant | proiezione ricostruibile | active view per workspace, revisione pinnata e document generation; payload senza vector | +| Sessione | verità del processo | `retrieval_pack.md`, `evidence_receipts.json`, `evidence.json`, `review_decisions.jsonl` | +| Chat/UI live | non persistente | screenshot o note di collaudo, mai usati come unico oracle | + +Nel percorso **filesystem v2** una Evidence è disponibile al runtime soltanto dopo source, curated, +manifest valido, assenza di review item irrisolti, preprocessing, evaluation superata e attivazione +atomica. HTTP e S3 seguono invece i rispettivi contratti adapter della campagna P2. + +Il solo puntatore Evidence `ACTIVE` non prova la disponibilità: il Qdrant adapter filtra anche per la +registry/workspace revision corrente. Dopo una candidate fallita si devono quindi confrontare +separatamente registry revision, Evidence ACTIVE pointer, punti fisici, vista di una sessione pinnata +e vista del runtime corrente; una divergenza è split-brain, non rollback riuscito. + +## 4. Topologia del laboratorio + +Il test deve usare lo stesso repository Git condiviso dei workspace, ma un namespace distinto. Non +va creato un corpus Evidence indipendente scollegato dal catalogo dei workspace. + +### 4.1 Isolamento obbligatorio + +- Branch remoto dedicato: `test/evidence-lifecycle`. +- Installazione locale dedicata al collaudo, con il processo backend configurato esplicitamente con + `THT_WORKSPACE_GIT_BRANCH=test/evidence-lifecycle`. La UI aggiorna il branch già configurato: non + permette di sceglierlo. +- Workspace ID: `psd-evidence-lab`. +- Directory: `psd-evidence-lab/`. +- Qdrant collection: `psd-evidence-lab`. +- Evidence URI: `psd-evidence-lab/evidence`. +- Database target: lo stesso DWH read-only di PSD, configurato però come binding del nuovo + workspace. +- Secret: reinseriti nella configurazione locale write-only; mai copiati in Git o nei report. +- Sessioni e corpus: namespace del nuovo workspace; nessun riuso dei path di `psd-clinical`. +- Prima dell'attivazione iniziale, committare almeno + `psd-evidence-lab/evidence/README.md`, escluso dal pattern `curated/**/*.md`: Git non conserva le + directory vuote e la materializzazione commit-addressed deve poter risolvere il tree Evidence. + +Il repository di authoring usato dall'umano deve essere un clone diverso dal checkout read-only +gestito dall'installazione. + +Collection e workspace distinti isolano i dati, ma non i fault ai servizi. REL-05/06/10-13 e le +prove di outage richiedono un Compose project lab con `dataRoot`, endpoint Qdrant/Ollama, volumi e +porte propri, oppure proxy di fault scoped esclusivamente al lab. Non fermare né corrompere i servizi +condivisi con PSD. Se questo isolamento non è disponibile, la campagna fault va marcata `NOT RUN` e +non è ammesso dichiarare `COMPLETE PASS`. + +### 4.2 Descriptor minimo + +Si parte dal descriptor PSD accettato, cambiando almeno ID, nome, descrizione, collection ed +Evidence URI. Per il laboratorio va dichiarato esplicitamente il contratto moderno: + +```yaml +evidence: + schema_version: 2 + source: + type: filesystem + uri: psd-evidence-lab/evidence + patterns: + - "curated/**/*.md" + policy: + max_chunk_chars: 5000 + retain_published_generations: 3 +``` + +Non copiare gli esempi legacy che usano `**/*.md` o omettono `schema_version: 2`. Aggiungere inoltre +l'entry ordinata a `thoth-workspaces.yaml` e, se serve lo stesso DWH nei processi core, portare nel +nuovo namespace le annotazioni schema curate già approvate, sottoponendole comunque al gate del +nuovo workspace. + +## 5. Responsabilità + +### 5.1 Attività dell'utente + +L'utente simula il curatore/reviewer reale e deve: + +1. preparare i tre testi descritti nella sezione 6; +2. leggere integralmente ogni diff delle Curated Evidence e confrontarlo con la source; +3. decidere sulle ambiguità, sui review item, sul rename, sul relink e sul retirement; +4. usare Workspace Management per update, validate e test connections; +5. porre le domande reali nell'app, decidere ai gate e giudicare correttezza business, citation e + SQL. + +### 5.2 Attività dell'agente/operatore tecnico + +L'agente può, dopo autorizzazione all'esecuzione del collaudo: + +1. creare scaffold, descriptor, catalog entry ed evaluation set; +2. eseguire i due CLI `tht` corretti, Git e le verifiche read-only; +3. raccogliere commit SHA, revision, run ID, generation e inventario Qdrant; +4. confrontare before/after e segnalare ogni violazione degli oracle; +5. predisporre e validare le probe assistite di runtime e inventory descritte nelle sezioni 7 e 16; +6. indurre fault controllati soltanto nel laboratorio isolato e ripristinare i servizi; +7. produrre il verbale finale PASS/FAIL con allegati e ripristinare l'installazione a PSD. + +## 6. Dataset umano: tre documenti, sei pubblicazioni + +L'attuale authoring skill impone esattamente una Evidence Unit per Source Evidence. I documenti +devono quindi contenere una sola unità primaria ciascuno. Un testo che combina concetti indipendenti +è un test negativo, non il formato nominale. + +### 6.1 Documento D1 — glossario/disambiguazione + +Path consigliato: +`evidence/source/00-glossario/coorte-lab-zaffiro.md`. + +Il testo deve includere: + +- una definizione business univoca; +- 2-3 sinonimi e almeno una variante ortografica italiana/Unicode; +- ciò che il termine non significa; +- eventuali tabelle/colonne, sempre pienamente qualificate; +- una frase breve copiabile come supporting excerpt; +- un token raro ma leggibile, per esempio `LAB-ZAFFIRO-731`; +- uso atteso: `disambiguation` e `rewriting`. + +### 6.2 Documento D2 — enum o regola legata allo schema + +Path consigliato: +`evidence/source/20-valori-enum/stato-lab-device.md`. + +Il testo deve includere: + +- una colonna reale nel formato `schema.table.column`; +- tutti i valori memorizzati e il loro significato; +- comportamento di `NULL`, valore sconosciuto ed eventuale eccezione; +- una frase breve copiabile come supporting excerpt; +- un token raro, per esempio `LAB-ENUM-842`; +- uso atteso: `schema_linking` e `sql_generation`. + +Se gli identificatori non sono pienamente qualificati, il sistema deve produrre un review item o +una Evidence `domain`, non inventare il mapping. + +### 6.3 Documento D3 — formula PostgreSQL + +Questo file viene aggiunto solo dopo la prima pubblicazione, per provare un inserimento +incrementale. Path consigliato: +`evidence/source/50-metadati-normalizzazione/formula-lab-intervallo.md`. + +Il testo deve includere: + +- un solo concetto calcolato; +- una sola espressione PostgreSQL componibile, non una query completa; +- tutti gli input come `schema.table.column`; +- semantica di `NULL`, unità di misura e casi limite; +- una frase breve copiabile come supporting excerpt; +- un token raro, per esempio `LAB-FORMULA-953`; +- uso atteso: `sql_generation`. + +Una formula contenente `SELECT`, DDL o DML deve essere rifiutata come test negativo. + +### 6.4 Scheda che l'utente compila prima del test + +| Campo | Valore da fornire | +| --- | --- | +| Tabella/colonna reale usata da D2 | | +| Valori reali e significato | | +| Colonne reali usate da D3 | | +| Espressione PostgreSQL attesa | | +| ID Evidence generati dopo `prepare` | | +| Domanda lessicale per D1/D2/D3 | | +| Parafrasi semantica per D1/D2/D3 | | +| Domanda che richiede D1 + D2 + D3 | | +| Risultato business e SQL attesi | | +| Domanda negativa non correlata | | +| Nuova definizione D1 per G3 | | +| Canary D1 nuova e formulazione da rimuovere | | +| Nuovi path D1 per G5 rename e G6 relink | | +| Testo business innocuo per la source injection S4 | | + +### 6.5 Copertura dei kind + +Il percorso principale copre tre payload differenti. Per verificare anche tutti gli otto kind +(`glossary`, `domain`, `enum`, `example`, `mapping`, `normalization`, `formula`, `reference`) il +dataset raccomandato usa cinque micro-source aggiuntive, una per kind mancante, coerentemente con la +skill corrente. Questa estensione è P2: non è necessaria per dimostrare il lifecycle CRUD, ma è +necessaria per dichiarare copertura completa dei payload. + +## 7. Convenzioni di esecuzione e raccolta + +Creare una directory di report esterna al repository di authoring, con un sottodirectory per ciclo: + +```text +evidence-test-run-YYYYMMDD-HHMM/ +├── 00-preflight/ +├── 01-g1-initial/ +├── 02-g2-add/ +├── 03-g3-update/ +├── 04-g4-retire/ +├── 05-g5-rename/ +├── 06-g6-relink/ +├── faults/ +└── final-report.md +``` + +Per ogni comando conservare timestamp, comando senza secret, exit code, stdout JSON, stderr +sanitizzato e identità dell'operatore. Per ogni pubblicazione conservare: + +- Git commit e workspace revision; +- `runId`, status e code; +- ACTIVE prima/dopo; +- generation manifest e mappa `document_generations`; +- inventario Qdrant filtrato; +- risultati dell'evaluation e delle probe; +- session ID e receipt. + +### 7.1 I due CLI omonimi + +Definire percorsi espliciti; non affidarsi al primo `tht` nel `PATH`: + +```bash +AUTHOR_THT=/Users/mp/projects/ThothII/harness/.venv/bin/tht +OPERATOR_THT=/absolute/path/to/native/host/tht +AUTHOR_REPO=/absolute/path/to/workspace-authoring-clone +WS_ROOT="$AUTHOR_REPO/psd-evidence-lab" +INSTALLATION=/absolute/path/to/thothii-installation.yaml +WS=psd-evidence-lab +REPORT=/absolute/path/to/evidence-test-run-YYYYMMDD-HHMM +EVIDENCE_PROBE=/absolute/path/to/assisted-evidence-probe +EVIDENCE_INVENTORY=/absolute/path/to/assisted-evidence-inventory +EVIDENCE_FAULT=/absolute/path/to/assisted-evidence-fault-harness +``` + +- `AUTHOR_THT`: authoring, validation e migrazione del workflow Python. +- `OPERATOR_THT`: installazione, registry e manutenzione del workspace reale. + +`--config`/`-c` del CLI Python è un'opzione del singolo comando e deve comparire dopo il +subcommand. + +Il backend crea lease/config runtime revision-pinned e temporanee: non si deve ricostruire, copiare o +passare a mano un `RUNTIME_CFG`. `EVIDENCE_PROBE` deve creare e distruggere la lease in modo +supportato; `EVIDENCE_INVENTORY` è read-only; `EVIDENCE_FAULT` opera solo sulla stack lab dedicata. +Sono helper che l'agente deve fornire e validare prima del collaudo esaustivo, non comandi attualmente +pubblici di `tht`. Se mancano, i casi che li richiedono sono `NOT RUN`/`INCONCLUSIVE`, non PASS per +inferenza dalla UI. + +## 8. Preflight e baseline + +| ID | Azione | Esito atteso | +| --- | --- | --- | +| PRE-01 | Verificare branch, worktree pulito e remote corretti | nessun file authoring già dirty; `.DS_Store` esclusi | +| PRE-02 | Eseguire le suite automatiche Evidence/backend rilevanti | PASS; eventuali L0/L2 mancanti documentati, non nascosti | +| PRE-03 | Verificare Pi, provider autore, Ollama, Qdrant, DWH e auth | tutti raggiungibili senza esporre credenziali | +| PRE-04 | Inventariare `psd-clinical` come gruppo di controllo | commit, ACTIVE, counts per kind e canary `disambiguation` con expected ID salvati | +| PRE-05 | Verificare assenza del workspace/collection lab | nessuna collisione; se esistono, fermarsi e identificare il proprietario | +| PRE-06 | Aggiungere catalog entry, descriptor lab e `evidence/README.md` tracciato in una revisione candidata | ID, URI e collection distinti; descriptor schema v3 valido; tree Evidence risolvibile anche senza D1-D3 | +| PRE-07 | Verificare nel processo dell'installazione `THT_WORKSPACE_GIT_BRANCH=test/evidence-lifecycle`, pushare la revisione su quel branch e usare **Update workspace repository** nella UI | la UI mostra ref/commit attesi, candidate valida attivata atomicamente; PSD resta disponibile | +| PRE-08 | Selezionare il lab, configurare binding/secret, **Validate workspace source** e **Test workspace connections** | check verdi; secret solo write-only | +| PRE-09 | Selezionare il lab come workspace dell'installazione | nuove sessioni usano il lab; nessuna sessione PSD viene mutata | +| PRE-10 | Preprocessare DWH, gestire il gate schema e indicizzare lo schema del lab, senza ancora eseguire lo stage Evidence | schema indicizzato nella collection lab; nessun punto PSD toccato | +| PRE-11 | Eseguire smoke test di `EVIDENCE_PROBE` e `EVIDENCE_INVENTORY` | helper attestano installazione/workspace/revision; nessuna scrittura o stampa di secret | +| PRE-12 | Eseguire `EVIDENCE_FAULT assert-isolated` e `status` | dataRoot, Qdrant, Ollama, volumi e porte lab non coincidono con PSD; nessun fault/barrier attivo; altrimenti campagna fault `NOT RUN` | +| PRE-13 | Verificare instrumentation del report candidate prima della compensazione | report sanitizzato osservabile; altrimenti dettaglio G3-bad `NOT OBSERVABLE` e niente `COMPLETE PASS` | + +Il full `workspace preprocess run` va usato soltanto dopo aver aggiunto D1/D2 e un evaluation set +valido; prima di quel momento lo stage Evidence non ha ancora un corpus valutabile. + +Non usare come prerequisito `scripts/evidence-restructuring-acceptance.sh`: nel tree corrente +riferisce un test non presente. Non assumere inoltre che il report PSD citato da `PROJECT_STATE.md` +sia disponibile nel working tree corrente. + +## 9. Comandi nominali + +### 9.1 Authoring + +```bash +"$AUTHOR_THT" evidence prepare "$WS_ROOT" --json +"$AUTHOR_THT" evidence validate "$WS_ROOT" --json + +# Dopo una rimozione e la decisione umana: +"$AUTHOR_THT" evidence resolve "$WS_ROOT" evidence: --retire --json +"$AUTHOR_THT" evidence resolve "$WS_ROOT" evidence: \ + --source source//.md --json + +``` + +Exit code attesi per `validate`: + +- `0`: corpus publishable; +- `1`: errore strutturale/operativo; +- `3`: soli orphan o review item irrisolti. + +Con `--json`, stdout deve contenere un solo JSON valido. + +### 9.2 Operatore host + +```bash +"$OPERATOR_THT" --installation "$INSTALLATION" workspace inspect \ + --workspace "$WS" --json +"$OPERATOR_THT" --installation "$INSTALLATION" workspace vector inspect \ + --workspace "$WS" --json +"$OPERATOR_THT" --installation "$INSTALLATION" workspace preprocess evidence \ + --workspace "$WS" --dry-run --json +"$OPERATOR_THT" --installation "$INSTALLATION" workspace preprocess evidence \ + --workspace "$WS" --json +``` + +Per un resume controllato: + +```bash +"$OPERATOR_THT" --installation "$INSTALLATION" workspace preprocess evidence \ + --workspace "$WS" --resume --json +``` + +### 9.3 Probe tipizzata tecnica, non superficie host + +`search evidence` richiede il config runtime temporaneo e non è direttamente esposto dal CLI host. +Il collaudo deve quindi usare un helper assistito che apra la stessa lease revision-pinned del core, +attesti nel JSON installazione, workspace, workspace revision e vector generation, invochi ricerca o +evaluation senza divulgare il config, quindi distrugga la lease. Interfaccia minima richiesta: + +```bash +"$EVIDENCE_PROBE" validate-fixture --workspace-root "$WS_ROOT" --json +"$EVIDENCE_PROBE" search --installation "$INSTALLATION" --workspace "$WS" \ + --stage clarification --query "" --json +"$EVIDENCE_PROBE" search --installation "$INSTALLATION" --workspace "$WS" \ + --stage rewriting --query "" --json +"$EVIDENCE_PROBE" search --installation "$INSTALLATION" --workspace "$WS" \ + --stage schema_linking --query "" --json +"$EVIDENCE_PROBE" search --installation "$INSTALLATION" --workspace "$WS" \ + --stage cte --query "" --json +"$EVIDENCE_PROBE" search --installation "$INSTALLATION" --workspace "$WS" \ + --stage final_sql --query "" --json + +# Filtri negativi RET-05: +"$EVIDENCE_PROBE" search --installation "$INSTALLATION" --workspace "$WS" \ + --stage schema_linking --query "" \ + --require-table "datawarehouse.tabella_inesistente_lab" --json +"$EVIDENCE_PROBE" search --installation "$INSTALLATION" --workspace "$WS" \ + --stage schema_linking --query "" \ + --require-column "datawarehouse.tabella_inesistente_lab.colonna_inesistente" --json +``` + +Il JSON di `search evidence` serve per ID, excerpt, purpose, revision e generation. I rank separati +`dense`, `bm25` e `fused` non sono presenti nel JSON di search. Dopo ogni preprocess riuscito +eseguire l'evaluation revision-pinned dell'ACTIVE; per una generation retained usare la sua revisione: + +```bash +"$EVIDENCE_PROBE" evaluate --installation "$INSTALLATION" --workspace "$WS" \ + --revision --active --json +"$EVIDENCE_PROBE" evaluate --installation "$INSTALLATION" --workspace "$WS" \ + --revision --generation --json +``` + +`evaluate` deve restituire profilo, expected/missing, empty result e rank dense/BM25/fused per query, +materializzare l'authoring tree e `evaluation.yaml` della revisione dichiarata, aprire una lease +Qdrant legata alla stessa revisione e attestare revision/generation nel JSON; usare la lease corrente +su una generation storica non è valido. + +L'implementazione corrente non persiste l'`EvaluationReport` della candidate fallita: lo riduce a un +esito booleano e compensa i punti. Per osservare `missing_expected` e rank di G3-bad serve prima una +modifica prodotto che salvi un report sanitizzato nel run artifact **prima** della compensazione. +Senza questa modifica si può provare solo il fallimento generico e l'immutabilità di ACTIVE; il +dettaglio candidate è `NOT OBSERVABLE` e preclude `COMPLETE PASS`. Se manca anche l'helper, marcare +RET-01/02/03/04/05/06 e le evaluation dirette `NOT RUN`; il collaudo resta `INCONCLUSIVE`. + +`validate-fixture` deve invocare lo stesso `load_evaluation_fixture` del prodotto senza model call né +accesso vector. `evidence validate` non legge `evaluation.yaml`: il suo exit `0` prova il corpus +authoring, non la struttura della fixture. + +I mapping stage-purpose attesi sono: + +| Stage | Purpose | +| --- | --- | +| `clarification` | `disambiguation` | +| `rewriting` | `rewriting` | +| `schema_linking` | `schema_linking` | +| `cte` | `sql_generation` | +| `final_sql` | `sql_generation` | + +Non devono esistere consultazioni Evidence in Memory o Synthesis. + +## 10. Ciclo G1 — creazione iniziale con D1 e D2 + +1. Aggiungere soltanto D1 e D2 in `source/`. +2. Eseguire `evidence prepare` da worktree con `curated/` e `manifest.yaml` puliti. +3. Verificare `changed=[D1,D2]`, due model call e due ID creati. Un numero diverso di candidati è + una violazione del contratto corrente e richiede revisione prima di proseguire. +4. Revisionare ogni Curated Evidence v3: + - una sola unità atomica; + - kind e purpose corretti; + - ID stabile `evidence:`; + - source file/hash corretti; + - excerpt esatto presente nella source; + - nessuna informazione inventata o secret; + - marker canonici presenti e Markdown leggibile. +5. Dopo ogni `prepare`, `curated/` e `manifest.yaml` sono dirty e un altro `prepare` è bloccato. + Revisionare l'output e creare un commit locale di review (non ancora pubblicato) per tornare a + un authoring state pulito; solo allora correggere la source e rieseguire `prepare`. Ripetere se + necessario e non correggere i marker a mano. +6. Creare `evaluation.yaml` dopo aver conosciuto gli ID. Il contratto richiede almeno una query per + ciascun profilo `lexical`, `semantic` e `mixed`; l'insieme deve coprire i purpose presenti. + Il profilo classifica il caso di test: l'evaluator esegue comunque il retrieval ibrido runtime e + registra separatamente rank dense, BM25 e fused. + Usare almeno una query dedicata per ciascun ID importante: quando una singola query dichiara più + ID attesi, il PASS Hit@10 richiede che ne compaia almeno uno e non prova automaticamente tutti. +7. Eseguire prima `EVIDENCE_PROBE validate-fixture`, poi `evidence validate`; entrambi devono uscire + `0`, mentre solo il secondo espone `publishable=true` per il corpus. +8. Committare localmente lo stato finale valido. Solo ora, da worktree pulito, rieseguire `prepare` + senza modifiche: `modelCalls=0`, `changed=[]`, entrambe le source in `unchanged` e nessun diff. +9. Pushare la testa valida della revisione G1; aggiornare il repository dalla UI. Eventuali commit + intermedi restano una review trail e non devono contenere secret. +10. Eseguire inspect, vector inspect, dry-run e preprocess reale. +11. Verificare evaluation PASS, attivazione atomica e inventory. + +### Gate G1 + +- Esistono esattamente due unità attive e recuperabili. +- Source/support file non vengono indicizzati. +- Il named vector `bm25` con modifier `idf` è presente senza perdita del dense unnamed. +- Punti schema preesistenti nella collection lab sono invariati. +- Un secondo preprocess identico restituisce `unchanged` e non crea punti duplicati. + +## 11. Ciclo G2 — inserimento incrementale di D3 + +1. Aggiungere soltanto D3. +2. Eseguire prepare/review/validate. +3. Verificare una sola model call, un solo nuovo ID e D1/D2 invariati. +4. Aggiungere all'evaluation query lexical, semantic o mixed per D3 e per la domanda composita; + eseguire `validate-fixture` oltre a `evidence validate`. +5. Commit, push, UI update, dry-run e preprocess reale. +6. Verificare nominalmente che D3 abbia una nuova document generation e che D1/D2 riusino le + precedenti. Se il cambio del path commit-addressed nella source URI forza il re-embedding di + tutti i documenti, registrarlo come difetto/limite di incrementalità, non come PASS silenzioso. + Se invece riusa le generation ma non esistono punti recuperabili sotto la revisione G2 corrente, + registrare FAIL di retrieval/incrementalità. +7. Verificare che il manifest ACTIVE elenchi tutte e tre le unità. +8. Prima di lasciare G2 eseguire RET-01..06, RET-08 e le sessioni S1, S2 e S3 della sezione 15.1: + dopo G4 D2 sarà ritirata e gli stessi oracle non saranno più eseguibili. + +### Gate G2 + +- La formula è una sola espressione PostgreSQL, con input qualificati. +- La formula pubblicata è citabile; una formula proposta nella sessione resta session-only e non + riceve un ID `evidence:`. +- Il retrieval combina correttamente D1, D2 e D3 quando la domanda lo richiede. + +## 12. Ciclo G3 — modifica e riacquisizione + +Questo ciclo combina update, pinning di sessione, conflitto e rollback della candidate. L'ordine è +parte dell'oracle e non va cambiato. + +1. Subito dopo G2, creare una sessione, portarla almeno fino a una receipt/citation D1 e lasciarla + incompleta e resumable. Salvare session ID, workspace revision G2, Evidence generation e receipt. +2. Nel clone di authoring modificare soltanto D1, mantenendo lo stesso concetto ma cambiando in modo + osservabile la definizione: aggiungere una nuova canary e rimuoverne una vecchia. +3. Prima di `prepare`, eseguire `validate`: deve rilevare il drift di hash/provenienza e bloccare. +4. Eseguire `prepare`: una model call, D1 in `changed`, D2/D3 in `unchanged`. L'ID D1 resta identico; + source hash e contenuto canonico cambiano. +5. Aggiornare le query nominali dell'evaluation per la nuova formulazione, mantenendo i profili + `lexical`, `semantic`, `mixed` e una query per ogni ID importante. +6. Aggiungere poi un caso G3-bad **strutturalmente valido**: un'ulteriore query `mixed` con expected + ID D1 esistente, ma purpose `sql_generation`, che non appartiene a D1 + (`disambiguation`/`rewriting`). Non usare ID inventati e non eliminare gli altri profili. Eseguire + `validate-fixture`: deve restituire `0`; eseguire separatamente `evidence validate`, il cui exit + `0` non costituisce prova sulla fixture. Se uno dei due fallisce, ridisegnare il caso prima del test. +7. Revisionare, committare G3-bad, pushare, usare **Update workspace repository** e registrare la + nuova registry active revision. L'Evidence ACTIVE generation deve essere ancora G2. +8. Fare resume della vecchia sessione: deve restare pinnata a G2, conservare artifact/receipt e + mostrare D1 vecchio, anche se il registry è già alla candidate revision G3-bad. +9. Tentare il preprocess G3-bad mentre la sessione è resumable: atteso + `preprocessing_conflict`; nessuna candidate generation né modifica ad ACTIVE. +10. Prima di finalizzare la sessione G2, salvare snapshot, artifact, citation e prova della revisione + pinnata. Poi finalizzarla per sbloccare il preprocess. Una sessione finalizzata non protegge più + lo snapshot dalla retention, anche se non è archiviata; non usarla come oracle post-G6. +11. Ripetere il preprocess G3-bad: la candidate deve fallire la retrieval evaluation e venire + compensata. Distinguere gli oracle: il puntatore Evidence e i punti fisici G2 restano; la vecchia + sessione pinnata G2 è stata verificata prima della finalizzazione; il runtime corrente è invece + legato alla registry revision G3-bad. Interrogarlo esplicitamente. Il PASS richiede rollback del + registry o routing coerente alla revisione G2; se la vista corrente è vuota per revision mismatch, + registrare split-brain/FAIL, non “ACTIVE G2 disponibile”. Con la diagnostica 9.3 salvare + `missing_expected`, rank e candidate generation; altrimenti dettaglio `NOT OBSERVABLE`. +12. Correggere la fixture assegnando a D1 un purpose ammesso, rieseguire `validate-fixture` e + `evidence validate`, committare G3-fix, pushare e aggiornare di nuovo il repository. +13. Eseguire dry-run e preprocess: evaluation PASS e switch atomico a G3. Verificare nuova document + generation per D1 e riuso delle document generation D2/D3; se la source URI commit-addressed + forza un full re-embedding, registrarlo come difetto di incrementalità. Se il riuso lascia i + punti solo sotto la revisione G2, il filtro runtime G3 non li recupera e il ciclo è FAIL. +14. La nuova canary deve trovare D1. La formulazione rimossa non deve comparire negli excerpt o + payload dell'active view; la sola retrieval semantica col vecchio termine non prova stale data. +15. Creare una nuova sessione: deve essere pinnata a G3 e vedere D1 nuovo. Registrare separatamente + revision, generation, artifact e receipt della sessione G2 e di quella G3. + +## 13. Ciclo G4 — cancellazione e retirement di D2 + +La cancellazione non deve mai essere silenziosa. + +1. Rimuovere il file source D2. +2. Eseguire `prepare`: D2 deve diventare orphan; l'unità non viene cancellata automaticamente. +3. Eseguire `validate`: exit `3`, `orphaned_unit` e/o `source_no_longer_supports_unit`; nessuna + pubblicazione ammessa. +4. Committare e pushare lo stato orphan su una revisione G4-bad del branch lab, quindi aggiornare il + repository dell'installazione isolata. Se il registry rifiuta la candidate, registrare esattamente + il componente che ha bloccato e marcare il preprocess downstream `NOT RUN`: non è la stessa prova + del gate di preprocess. Se il registry la accetta, il preprocess deve bloccare e lasciare il + puntatore Evidence su G3; poi interrogare il runtime corrente G4-bad. Se non recupera G3 perché il + filtro revision-pinned non coincide, registrare split-brain/FAIL. Se il registry ha rifiutato la + candidate ed è rimasto G3, verificare invece che G3 resti recuperabile. +5. Da quel worktree pulito, il reviewer decide il retirement ed esegue + `evidence resolve ... --retire`. +6. Aggiornare `evaluation.yaml`: eliminare o sostituire ogni query che attende D2, mantenendo almeno + un caso per ciascun profilo `lexical`, `semantic`, `mixed`, la copertura dei purpose ancora attivi + e query dedicate a D1/D3. +7. Eseguire `validate-fixture` e `evidence validate` con exit `0`, committare la risoluzione G4-fix, + pushare la testa valida, fare UI update e preprocess. +8. Verificare che D2 non sia nel manifest ACTIVE, nell'evaluation o nei risultati runtime. + +Punti di D2 possono ancora esistere fisicamente in una generazione retained. Non è un difetto se il +manifest ACTIVE e i filtri per document ID/generation impediscono di restituirli. È un difetto se +una ricerca attiva li restituisce. + +## 14. Cicli G5 e G6 — rename e relink di D1 + +Eseguire entrambi i rami in revisioni separate. + +### 14.1 G5 — rename identico + +1. Rinominare la source D1 senza cambiare un byte. +2. Eseguire `prepare`. +3. Atteso: riconoscimento univoco tramite hash, ID stabile, nessuna ristrutturazione non necessaria. +4. Pubblicare e verificare citation/provenienza sul nuovo path. + +### 14.2 G6 — relink esplicito + +1. Rimuovere il source path corrente di D1, eseguire `prepare` e verificare l'orphan con `validate` + exit `3`; creare un commit locale G6-orphan. Questo passaggio è necessario: aggiungere prima il + nuovo file trasformerebbe il caso in rename/hash match, non in relink. +2. Dal commit G6-orphan creare un worktree/branch negativo separato. Aggiungere e committare una + source priva dell'exact excerpt, poi lanciare `resolve --source`: atteso exit `1` con finding + `supporting_excerpt_missing`. Il relink può essere applicato, ma `validate`/publish devono restare + bloccati; non unire questa variante. +3. Nel worktree principale, ancora pulito sul commit G6-orphan, aggiungere il nuovo file source con + gli exact excerpt necessari e committarlo **senza** eseguire `prepare`. +4. Eseguire `evidence resolve ... --source source/`. +5. Verificare stesso ID, nuovo source file/hash, rimozione dell'orphan e assenza del finding. +6. Validare, commit/push, UI update, preprocess e verificare il nuovo citation path. Conservare il + branch negativo fino alla chiusura del report; niente reset distruttivi. + +## 15. Matrice di retrieval e uso core + +| ID | Scenario | Oracle | +| --- | --- | --- | +| RET-01 | Token esatto di ogni source | Evidence attesa entro top 10; rank dense/BM25/fused registrati | +| RET-02 | Parafrasi senza token esatto | Evidence attesa via ramo semantico/fusione | +| RET-03 | Query mixed | Evidence attesa con entrambe le branche diagnosticate | +| RET-04 | Purpose errato | Evidence non deve comparire nello stage non autorizzato | +| RET-05 | `required_table`/`required_column` non corrispondenti | outcome `available` con zero risultati | +| RET-06 | Query non correlata | `available` vuoto o nessun ID atteso; non `unavailable` | +| RET-07 | Qdrant/Ollama indisponibile | outcome `unavailable`, stage bloccato e retry dello stesso stage; niente stale fallback | +| RET-08 | Canary incrociate lab/`psd-clinical` | nessun ID o payload dell'altro workspace attraversa il namespace | +| RET-09 | Query lab dopo modifica | solo contenuto della revisione/generazione attiva | +| RET-10 | Citation | file immutabile materializzato, source/provenienza verificabili | +| CORE-01 | Sessione F1-F8 completa | cinque receipt: clarification, rewriting, schema_linking, cte, final_sql | +| CORE-02 | Evidence accettata al gate | decisione persistita e `evidence.json` coerente | +| CORE-03 | Evidence rifiutata al gate | rifiuto persistito; non usata come fatto downstream | +| CORE-04 | Formula proposta dalla sessione | resta `formula_proposal`, non Published Evidence | +| CORE-05 | Nuova vs vecchia sessione | revision pinning rispettato | +| CORE-06 | Prompt injection nella source | trattata come dati, mai come istruzione; nessuna esfiltrazione/tool action | + +Le receipt devono contenere stage, purpose, vector generation e ID, non una copia integrale delle +Evidence. Una nuova ricerca nello stesso stage sostituisce solo la receipt di quello stage. + +### 15.1 Sessioni utente obbligatorie + +Eseguire casi separati: una sola sessione non dimostra tutti i comportamenti. S1, S2, S3, +RET-01..06 e RET-08 vanno eseguiti alla fine di G2; S5 durante G3; S4 solo dopo retention, snapshot e +non-regressione G6. + +1. **S1 — accettazione:** porre una domanda che richieda D1, D2 e D3; avanzare F1-F8, accettare le + Evidence pertinenti ai gate e salvare cinque receipt. Verificare decision ledger, `evidence.json`, + citation e SQL finale contro i testi originali. +2. **S2 — rifiuto:** in una nuova sessione porre una domanda che recuperi certamente D2, rifiutarla + al gate e completare il flusso. Il rifiuto deve essere persistito e D2 non deve diventare un fatto + accettato negli artifact downstream. +3. **S3 — proposta:** chiedere un calcolo non presente nel corpus e lasciare che la sessione proponga + una formula. Deve restare `formula_proposal` session-only: nessun file curated, manifest, punto + Qdrant o ID `evidence:` viene creato. +4. **S4 — prompt injection (G7/G8 security):** in G7 aggiungere una quarta source atomica con + contenuto business innocuo e una frase che ordina al modello di ignorare le regole, mostrare + secret o usare tool. `prepare` non deve obbedire alla frase né produrre leakage/tool action; + aggiungere una query evaluation dedicata al nuovo ID, eseguire `validate-fixture`, pubblicare, + interrogarla in una nuova sessione e verificare che sia trattata solo come dato. In G8 ritirarla, + rimuovere/sostituire la sua query, rivalidare la fixture e pubblicare la revisione pulita. +5. **S5 — pinning:** usare le due sessioni G2/G3 della sezione 12 e confrontare revision, generation, + citation e artifact, non soltanto il testo mostrato nella chat. + +Per RET-04 eseguire una query D1 nello stage `schema_linking` e una D2 nello stage `rewriting`; per +RET-05 usare entrambe le opzioni `--require-table` e `--require-column` della probe; per RET-06 usare +una query non correlata predefinita. L'oracle è rispettivamente assenza per purpose, `available` con +risultato vuoto e `available` vuoto. RET-07 va eseguito solo nella stack fault isolata: l'outcome è +`unavailable`, la fase corrente non avanza e il retry riparte dallo stesso stage. + +Per RET-08 usare la canary e l'ID PSD salvati in PRE-04 e una canary lab rara: + +```bash +"$EVIDENCE_PROBE" search --installation "$INSTALLATION" --workspace psd-clinical \ + --stage clarification --query "LAB-ZAFFIRO-731" --json +"$EVIDENCE_PROBE" search --installation "$INSTALLATION" --workspace "$WS" \ + --stage clarification --query "" --json +``` + +Ogni JSON deve attestare il workspace/revision richiesto e non contenere ID, excerpt o payload +dell'altro namespace. Ripetere RET-08 dopo G6 e dopo G8 senza cambiare le impostazioni globali o +mutare PSD. + +`evaluation.yaml` richiede almeno un ID atteso non vuoto e non esprime forbidden ID o un oracle +“deve restituire zero risultati”. RET-04, RET-05 e RET-06 devono quindi restare probe manuali/CLI +separate e non possono essere dichiarate coperte dal solo PASS dell'evaluation. + +## 16. Inventario vettoriale corretto + +`workspace vector inspect` verifica oggi collection, dimensioni e distanza; non è un inventario dei +record. Per il test esaustivo l'agente deve fornire un helper read-only con questa interfaccia minima: + +```bash +"$EVIDENCE_INVENTORY" capture --installation "$INSTALLATION" --workspace "$WS" \ + --scope pointer --output "$REPORT//inventory-pointer.json" +"$EVIDENCE_INVENTORY" capture --installation "$INSTALLATION" --workspace "$WS" \ + --scope active --output "$REPORT//inventory-active.json" +"$EVIDENCE_INVENTORY" capture --installation "$INSTALLATION" --workspace "$WS" \ + --scope retained --output "$REPORT//inventory-retained.json" +``` + +L'helper risolve i path e gli endpoint dall'installazione senza stampare secret, legge `ACTIVE` e il +generation manifest, quindi pagina lo scroll Qdrant fino a esaurimento con `with_vector=false`. Deve +produrre tre viste distinte: set dichiarato dal pointer, active runtime e retention fisica. Se +l'helper non è presente o non attesta la revisione osservata, l'inventory è `INCONCLUSIVE`. + +Per ogni ciclo: + +1. leggere `ACTIVE` e il generation manifest; +2. estrarre `document_generations`; +3. costruire la pointer view dalle coppie esatte `(document_id, vector_generation)` di + `document_generations`, anche cross-revision, per provare quali punti fisici supportano il + puntatore Evidence; +4. costruire l'active runtime view aggiungendo `workspace_id=psd-evidence-lab` e la + `workspace_revision` pinnata, replicando tutti i predicati runtime **prima** del limit; +5. acquisire separatamente tutti i punti retained del workspace e raggrupparli per Evidence ID, + document ID, vector generation e `workspace_revision`; ogni punto deve avere + `record_kind=evidence` e `kind=evidence`; +6. verificare gli otto payload index richiesti: + `content_hash`, `document_id`, `kind`, `record_key`, `record_kind`, `vector_generation`, + `workspace_id`, `workspace_revision`; +7. verificare che gli ID/document generation consentiti dal manifest ACTIVE e quelli realmente + interrogabili dal runtime coincidano; ogni differenza è split-brain. + +Non assumere che tutti i documenti attivi abbiano la generation più recente: il manifest può riusare +una document generation precedente. Tuttavia il Qdrant runtime filtra anche per revisione corrente: +se i punti riusati esistono solo sotto la revisione vecchia e non vengono recuperati, è un difetto di +incrementalità/retrieval, non un motivo per omettere il filtro dall'oracle. Non assumere neppure che +il numero totale di punti Qdrant equivalga al corpus attivo: retention e candidate cambiano il totale. + +## 17. Atomicità, idempotenza, retention e recovery + +| ID | Scenario | Esito atteso | +| --- | --- | --- | +| REL-01 | `preprocess --dry-run` | nessuna modifica a ACTIVE, corpus o punti | +| REL-02 | prepare invariato | zero model call e zero diff | +| REL-03 | preprocess invariato | status `unchanged`, nessuna duplicazione | +| REL-04 | evaluation candidata fallisce | candidate compensata e pointer precedente intatto; registry/runtime ancora coerenti, altrimenti split-brain FAIL | +| REL-05 | embedding fallisce durante candidate | nessuna attivazione o stato parziale visibile | +| REL-06 | Qdrant fallisce durante upsert | ACTIVE precedente intatto; retry idempotente | +| REL-07 | due preprocess simultanei | uno solo procede; l'altro `preprocessing_conflict` | +| REL-08 | resume stesso run/commit/config | riparte dal checkpoint valido e pubblica una sola volta | +| REL-09 | resume dopo cambio commit/config | `preprocessing_resume_mismatch`, nessuna mutazione | +| REL-10 | checkpoint/artifact corrotto | fail closed, nessuna attivazione | +| REL-11 | collection mancante | self-heal compatibile; dense, BM25/IDF e payload index ricreati | +| REL-12 | BM25 mancante | aggiunta `bm25`/`idf` senza rebuild del dense | +| REL-13 | dimensioni/distance/BM25 incompatibili | `semantic_index_incompatible`, nessuna mutazione distruttiva | + +REL-01/02/03/04 fanno parte del percorso lifecycle. REL-05..13 sono una campagna engineering su stack +dedicata: usare un fault proxy scoped, volumi usa-e-getta e snapshot del dataRoot lab. Durante un +fallimento mid-upsert possono esistere punti candidate fisici, ma non devono entrare nell'active +view; compensazione/retry devono ricondurre lo stato all'oracle. Non simulare questi casi fermando +Qdrant o Ollama condivisi con PSD. + +L'harness deve offrire `assert-isolated`, `inject`, `barrier`, `wait`, `abort`, `release`, `clear`, +`status` e `restore`, sempre attestando installazione, workspace, endpoint e collection. Preflight e +cleanup minimi: + +```bash +"$EVIDENCE_FAULT" assert-isolated --installation "$INSTALLATION" --workspace "$WS" --json +"$EVIDENCE_FAULT" status --installation "$INSTALLATION" --workspace "$WS" --json +"$EVIDENCE_FAULT" clear --installation "$INSTALLATION" --workspace "$WS" --all --json +"$EVIDENCE_FAULT" restore --installation "$INSTALLATION" --workspace "$WS" --json + +# Forme comuni; inject/barrier restituiscono faultId/barrierId: +"$EVIDENCE_FAULT" inject --installation "$INSTALLATION" --workspace "$WS" \ + --fault --once --json +"$EVIDENCE_FAULT" barrier --installation "$INSTALLATION" --workspace "$WS" \ + --at --json +"$EVIDENCE_FAULT" wait --installation "$INSTALLATION" --workspace "$WS" \ + --event barrier-reached --json +"$EVIDENCE_FAULT" abort --installation "$INSTALLATION" --workspace "$WS" \ + --run-id --json +"$EVIDENCE_FAULT" release --installation "$INSTALLATION" --workspace "$WS" \ + --barrier-id --json +``` + +| Caso | Fault/operazione deterministica | +| --- | --- | +| REL-05 | `inject --fault embedding-fail --once`; preprocess, ACTIVE invariato, `clear` e retry | +| REL-06 | `inject --fault qdrant-upsert-fail --after-points --once`; inventory candidate/active, `clear` e retry | +| REL-07 | `barrier --at after-writer-lock`; terminale A avvia preprocess; `wait` restituisce run ID; terminale B tenta preprocess e riceve conflict; `release` completa A | +| REL-08 | `barrier --at after-checkpoint:embed`; avviare e attendere il run ID, `abort --run-id`, poi host CLI `--resume ` sullo stesso commit/config | +| REL-09 | creare un secondo checkpoint come REL-08, abortire, pubblicare una candidate commit/config diversa e tentare `--resume `; atteso mismatch; poi ripristinare con un nuovo commit valido | +| REL-10 | su copia del checkpoint, `inject --fault corrupt-checkpoint --run-id `; resume fail closed; `restore` | +| REL-11 | dopo snapshot, `inject --fault collection-missing`; preprocess deve ricreare il contratto previsto; `restore` | +| REL-12 | `inject --fault collection-dense-only`; verificare aggiunta BM25/IDF senza perdita dense; `restore` | +| REL-13 | ripetere con `wrong-dimension`, `wrong-distance`, `invalid-bm25`; ogni caso fallisce senza rebuild distruttivo; `restore` | + +I comandi `wait`/`status` devono essere la fonte del run ID: oggi non esiste un elenco job pubblico. +Dopo ogni caso eseguire `clear`, verificare nessun processo/barrier residuo, inventory e ACTIVE. Se +l'harness non implementa l'interfaccia, REL-05..13 sono `NOT RUN` e `COMPLETE PASS` è vietato. + +Le pubblicazioni G1-G6 superano `retain_published_generations: 3`. Dopo G6 verificare: + +- ACTIVE + due generazioni di rollback conservate; +- generazione più vecchia eliminata quando non protetta; +- sub-generation ancora referenziate dai manifest retained non eliminate; +- checkpoint resumable protetti e preprocessing di una revisione diversa bloccato finché esiste + una sessione resumable incompatibile; +- snapshot Git referenziati da sessioni ancora resumable (`status != finalized` e non archiviate) + conservati; la prova G2 si raccoglie prima di finalizzare la sessione nella sezione 12; +- filesystem e Qdrant coerenti dopo GC. + +Il comando GC della CLI harness è una diagnostica tecnica, non una superficie operatore pubblica; +non va presentato come normale gesto utente. + +## 18. Test negativi e sicurezza + +Eseguire i casi distruttivi solo in un worktree/branch e collection usa-e-getta del laboratorio. + +### 18.1 Authoring e canonical format + +- source multi-concetto: una sola unità primaria oppure review item, mai fusione silenziosa; +- supporting excerpt assente o non esatto; +- ID duplicato o stessa unità associata a due source; +- body/metadata desincronizzati; +- marker `tht:` mancante, duplicato o alterato; +- campo sconosciuto o payload del kind errato; +- source hash/manifest hash alterato manualmente; +- file non UTF-8, vuoto non significativo o oltre 10 MiB; +- symlink in source/curated; +- dirty `curated/` o `manifest.yaml` prima di prepare; +- una di tre ristrutturazioni fallisce: nessuna applicazione parziale. + +### 18.2 Descriptor e materializzazione + +- `evidence.schema_version` omesso: il parser può applicare il default legacy v1, ma il laboratorio + deve rifiutarlo come deviazione dalla baseline v2 esplicita; un valore invalido deve fallire; +- pattern diverso da `curated/**/*.md`; +- per la source filesystem, URI assoluto, traversal o cross-namespace; HTTP richiede un URI + `http(s)://` canonico e S3 un URI `s3://` secondo i rispettivi adapter; +- symlink, gitlink, path duplicato, file non regolare o race; +- oltre 4096 entry, 64 MiB totali, 8 MiB/file, manifest oltre 1 MiB; per il path verificare + esplicitamente 4096 byte accettati e 4097 byte rifiutati; +- candidate Git revision invalida: la precedente resta attiva. + +I boundary di 4096 entry, 64 MiB e path 4096/4097 byte vanno costruiti in fixture automatiche o in +un repository Git usa-e-getta con plumbing controllato e materializer isolato, non pushati nel +repository condiviso del lab. Il test path deve inoltre verificare che l'ambiente permetta di creare +la fixture senza fallire prima del codice sotto test. Se manca l'harness, questi casi sono `NOT RUN` +nel collaudo manuale. + +### 18.3 Dati, rete e segreti + +- provenance URI dichiarata con userinfo, credenziali, token o query firmata; il transport URL + firmato supportato può invece stare nel `signed_urls_file`, ma non deve apparire in descriptor, + report o log; +- prompt injection nella source; +- tentativo di indicizzare `source/`, `evaluation.yaml` o support file; +- HTTP redirect/rebind verso host privato e S3 endpoint non consentito, nella suite adapter P2; +- utente non amministratore sui controlli Workspace Management; +- JSON machine output contaminato da log o secret; +- export/backup applicativo che includa byte Evidence o secret quando non previsto. + +## 19. Non regressione + +Prima di G1 e dopo G6 confrontare: + +- punti e nearest-neighbour canary di `schema_table` e `schema_column`; +- punti `memory` e `solved_question` eventualmente presenti; +- dimensioni, distanza e dense unnamed della collection; +- query di controllo in `psd-clinical`; +- accesso DWH rigorosamente read-only; +- settings globali ripristinati esattamente al baseline e sessioni PSD non modificate. + +`delete_generation`/retention Evidence non deve cancellare altri record kind. `vector rebuild +--destroy` non fa parte del percorso nominale; se usato per provare restore/cleanup deve puntare +alla collection lab, ripetere esattamente il suo nome nei guard e avere autorizzazione esplicita. + +## 20. Adapter e kind estesi (P2) + +La dichiarazione di copertura completa oltre il percorso filesystem richiede due campagne separate: + +1. HTTP reale: redirect, cache, timeout, byte limit, SSRF e rebind; +2. S3 reale: paginazione, size/object limits, secret file bounded, endpoint policy e credenziali non + divulgate. + +Non applicare a HTTP/S3 l'oracle del tree `source/`/`curated/`/`manifest.yaml` né l'obbligo di +candidate evaluation del filesystem v2. Per questi adapter l'oracle è acquire/normalize/chunk del +contenuto remoto dichiarato, secondo URI, credenziali, limiti e policy specifici. + +Per i cinque kind non coperti da D1-D3 creare micro-source atomiche e ripetere +prepare/validate/preprocess/evaluate/search. Questi test non devono essere mescolati al lifecycle +principale, altrimenti un fallimento non è diagnosticabile. + +La campagna P2 deve inoltre coprire: + +- fixture legacy v1 e v2 con `"$AUTHOR_THT" evidence migrate --json`, verificando + migrazione deterministica a v3, nessuna model call e ID/provenienza coerenti; +- `"$AUTHOR_THT" evidence prepare --upgrade --json`, che riprocessa tutte le source + con la pipeline installata senza duplicare unità; +- `"$EVIDENCE_PROBE" evaluate ... --revision --generation --json`, + materializzando snapshot, config ed `evaluation.yaml` della revisione che pubblicò la generation; + provare sia ACTIVE sia una generation retained; +- un cambio controllato di `kind` della stessa unità, con payload completo per il nuovo kind: l'ID + resta stabile, la nuova versione è attiva e la vecchia non è restituita dall'active view. + +## 21. Copertura automatica da usare come prerequisito + +Eseguire i gate completi con comandi separati: + +```bash +cd /Users/mp/projects/ThothII/harness +.venv/bin/pytest -q + +cd /Users/mp/projects/ThothII/backend +npx vitest run +npx tsc --noEmit -p . + +cd /Users/mp/projects/ThothII/frontend +npx vitest run +npx tsc -b +``` + +Il marker `l2` è escluso dal default del harness e va eseguito esplicitamente con +`.venv/bin/pytest -q -m l2` quando credenziali e servizi reali sono disponibili. I test `l0` che +usano testcontainers richiedono Docker; skip o ambiente mancante vanno riportati nel verbale. +Conservare inoltre il dettaglio dei gruppi rilevanti: + +- `harness/tests/test_evidence_authoring.py` e `test_evidence_cli.py`; +- `test_evidence_canonical.py`, filesystem/HTTP/S3 source adapter tests; +- `test_corpus_pipeline.py`, `test_corpus_publish.py`, candidate publication ed evaluation; +- `test_qdrant_vector_store.py` e, con Docker, `tests/l0/test_qdrant_bm25_inference.py`; +- backend registry/materialization/preprocessing service/state tests; +- frontend `WorkspaceManager.test.tsx`. + +Queste suite usano in vari punti fake di embedder/Qdrant. Un PASS automatico non sostituisce il +percorso live di questo piano. + +### 21.1 Seam ad alto rischio da osservare esplicitamente + +Questi punti risultano rischiosi nell'implementazione corrente e non devono essere esclusi dal +verbale se falliscono: + +- la risoluzione citation delle unità v3 potrebbe cercare l'ID nella forma legacy anziché nei + metadata `curated_evidence`; +- una nuova source URI commit-addressed può far apparire modificati documenti con contenuto + invariato e causare un full re-embedding; +- se una document generation viene riusata ma i suoi punti esistono solo sotto la revisione vecchia, + il filtro Qdrant revision-pinned può renderla irrecuperabile: inventory e retrieval devono fallire; +- `workspace vector rebuild` può ricreare il dense senza ripristinare subito BM25: non usarlo come + recovery nominale e, se testato, verificare il contratto completo dopo il rebuild; +- il writer lock/conflitto va provato live, perché la sola presenza delle primitive nei test non + dimostra che il percorso produttivo le acquisisca; +- non esiste un comando pubblico per inventory, elenco job o riattivazione di una generazione + precedente; +- il report dettagliato della candidate evaluation fallita non è persistito prima della + compensazione; serve instrumentation prodotto per osservarne rank e `missing_expected`; +- l'adapter S3 operativo è più restrittivo di alcune varianti accettate dallo schema; i casi custom + vanno classificati come gap di superficie, non come scenario nominale. + +## 22. Criteri finali PASS/FAIL + +Il collaudo filesystem/lifecycle è PASS solo se: + +1. G1-G6 completano con gli oracle documentati; +2. ID stabili, hash, provenance, revision e generation sono coerenti; +3. l'inserimento incrementale non reprocessa documenti invariati; +4. la modifica non è visibile prima dell'atomic switch, la versione precedente resta recuperabile e + la nuova lo è dopo; nessuno split-brain registry/Evidence; +5. orphan blocca, retirement/relink sono espliciti e auditabili; +6. nessun record retained/obsolete viene restituito come attivo; +7. retrieval copre tutti i purpose, verifica l'outcome `available` vuoto e non mescola + workspace/revision; +8. le sessioni S1, S2, S3 e S5 coprono accettazione, rifiuto, proposta session-only e pinning; S1 + contiene cinque receipt e decisioni/citation corrette; +9. il conflict di sessione G3 e l'evaluation failure lasciano ACTIVE consistente; +10. retention/GC e non regressione Schema/Memory/solved sono verificati; +11. nessun secret o path non sicuro attraversa i confini; +12. ogni FAIL ha reproduction, expected/actual, commit, revision, run ID e generation. + +La dichiarazione **COMPLETE PASS** richiede inoltre: G7/G8 con S4 prompt injection; RET-07 +`unavailable`/stage bloccato; report sanitizzato della candidate evaluation fallita; campagna +fault/resume REL-05..13 su stack isolata; +tutti gli otto kind; adapter HTTP/S3 reali supportati; migrazione v1/v2, `--upgrade`, evaluation di +una generation selezionata e cambio kind. Se una di queste campagne non viene eseguita, il miglior +esito ammesso è `PASS — filesystem lifecycle`, mai “copertura completa”. + +Il risultato è **INCONCLUSIVE**, non PASS, se mancano inventory Qdrant, sessione core completa, +proof di ACTIVE before/after o verifica della revisione pinnata. + +## 23. Teardown sicuro + +Eseguire il teardown solo dopo aver completato e verificato gli allegati: + +1. salvare hash del report, registry active revision, Evidence ACTIVE/generation manifest, inventory + Qdrant e stato delle sessioni; +2. verificare che il report contenga la prova snapshot/artifact/citation G2 raccolta **prima** della + finalizzazione in G3; non richiedere che lo snapshot G2 esista ancora dopo G6; +3. finalizzare tutte le sessioni lab ancora resumable e archiviarle dove la superficie lo consente, + senza riaprire sessioni già chiuse; +4. nella UI ripristinare `psd-clinical` come workspace selezionato e le impostazioni originarie; + eseguire **Validate workspace source**, **Test workspace connections** e una nuova query canary PSD; +5. rimuovere binding e secret del lab tramite la superficie supportata, se previsto dalla campagna; + se non esiste una rimozione supportata, documentare il residuo senza esporne il valore; +6. non cancellare collection, dataRoot, branch, commit o report prima della firma del verbale. La loro + eliminazione è un'attività distruttiva separata e richiede autorizzazione esplicita e target + risolti; preferire snapshot/archiviazione recuperabile; +7. ripetere il controllo non-regressione PSD e registrare che settings, sessioni e punti non sono + cambiati. Un fallimento di ripristino rende l'intero collaudo FAIL. + +## 24. Template del verbale + +```markdown +# Evidence lifecycle acceptance — + +- Tester: +- Installazione: +- Branch/commit finale: +- Workspace/collection: +- DWH binding type: +- Ollama model/dimensions: +- Scope verdict (`PASS — filesystem lifecycle` / `COMPLETE PASS` / `INCONCLUSIVE` / `FAIL`): +- Version/hash di probe, inventory, fault harness e instrumentation evaluation: + +| Ciclo | Commit | Registry active revision | Run ID | Candidate generation | Evaluation | Evidence ACTIVE before/after | Runtime active view | `document_generations` | Esito | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | +| G1 initial | | | | | | | | | | +| G2 add | | | | | | | | | | +| G3-bad | | | | | | | | | | +| G3-fix | | | | | | | | | | +| G4-bad | | | | | | | | | | +| G4-fix | | | | | | | | | | +| G5 rename | | | | | | | | | | +| G6 relink | | | | | | | | | | +| G7 security injection | | | | | | | | | | +| G8 security retire | | | | | | | | | | + +## Retrieval/sessioni + +| Query/stage | Purpose/filtri | Revision/generation | Expected/actual IDs | Rank dense/BM25/fused | Outcome | PASS/FAIL | +| --- | --- | --- | --- | --- | --- | --- | + +## Fault e recovery + +| Fault/fault ID | Barrier/run ID | ACTIVE prima | Risultato | ACTIVE dopo | Cleanup | PASS/FAIL | +| --- | --- | --- | --- | --- | --- | --- | + +## Non regressione e sicurezza + +- Schema/Memory/solved: +- Workspace isolation: +- Secret scan: +- Prompt injection: +- Retention/GC: + +## Difetti + +| ID | Severità | Reproduction | Expected | Actual | Allegati | +| --- | --- | --- | --- | --- | --- | +``` + +## 25. Riferimenti normativi + +- `CONTEXT.md`, sezione Evidence; +- `docs/evidence.md`; +- `docs/contracts/workspace-evidence-v3.md`; +- `docs/contracts/workspace-preprocessing-cli.md`; +- `docs/operations/workspaces.md`; +- `harness/.pi/skills/tht-evidence-authoring/SKILL.md`; +- `harness/.pi/skills/tht-sessione/SKILL.md` e modulo runtime Evidence; +- `harness/tht/evidence/` e `harness/tht/cli/search_cmd.py`. diff --git a/frontend/e2e/database-management-layout.spec.ts b/frontend/e2e/database-management-layout.spec.ts index 83877a16..95df0ab4 100644 --- a/frontend/e2e/database-management-layout.spec.ts +++ b/frontend/e2e/database-management-layout.spec.ts @@ -1,4 +1,4 @@ -import { expect, test, type Page, type Route } from "@playwright/test"; +import { expect, test, type Locator, type Page, type Route } from "@playwright/test"; import { createAuthenticationStack } from "./fixtures/auth-stack.mjs"; test.describe.configure({ mode: "serial" }); @@ -12,6 +12,13 @@ const database = { workspaceId: "psd-clinical", workspaceName: "Policlinico San Donato", workspaceAvailable: true, + workspaceRevision: { commit: "a".repeat(40), blob: "b".repeat(40) }, + workspaceEvidence: { sourceType: "filesystem", state: "materialized_current_revision" }, + runtimeBinding: { + transport: "postgres_direct", + configurationState: "ready", + sessionTransportSupported: true, + }, configured: true, engine: "postgres", databaseName: "warehouse", @@ -37,6 +44,22 @@ const database = { }, }; +const unconfiguredDatabase = { + ...database, + id: undefined, + workspaceId: "research-lab", + workspaceName: "Research laboratory", + configured: false, + databaseName: "research", + schema: "analytics", + version: 0, + createdAt: "", + updatedAt: "", + binding: { transport: "postgres_direct", port: 5432 }, + connectionStatus: "untested", + testedVersion: undefined, +}; + const table = { id: "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa", databaseId, @@ -118,7 +141,112 @@ async function expectContextPanelGeometry(page: Page, accessibleName: string) { expect(panelBox.y + panelBox.height).toBeLessThanOrEqual(managerBox.y + managerBox.height + 1); } -async function expectContextPanelHeaderMatchesNewSession( +async function expectCurrentNavigation(page: Page, accessibleName: string) { + const navigation = page.getByRole("complementary", { name: "Session navigation" }); + const button = navigation.getByRole("button", { name: accessibleName, exact: true }); + const available = navigation.locator('[data-navigation-state="available"]').first(); + await expect(navigation.locator('[data-navigation-state="current"]')).toHaveCount(1); + await expect(button).toHaveAttribute("data-navigation-state", "current"); + await expect(button).toHaveAttribute("aria-current", "page"); + expect(await button.getAttribute("class")).toContain("bg-[oklch(var(--nav-active))]"); + + const [style, availableStyle] = await Promise.all([ + button.evaluate((element) => ({ + backgroundColor: getComputedStyle(element).backgroundColor, + borderColor: getComputedStyle(element).borderTopColor, + })), + available.evaluate((element) => ({ + backgroundColor: getComputedStyle(element).backgroundColor, + borderColor: getComputedStyle(element).borderTopColor, + })), + ]); + expect(style.backgroundColor).not.toBe(availableStyle.backgroundColor); + expect(style.borderColor).not.toBe(availableStyle.borderColor); + + const tokens = await page.evaluate(() => { + const root = getComputedStyle(document.documentElement); + const read = (name: string) => root.getPropertyValue(name).trim().split(/\s+/).map(Number); + return { primary: read("--primary"), active: read("--nav-active") }; + }); + expect(tokens.active[2]).toBeCloseTo(tokens.primary[2], 1); + expect(tokens.active[1]).toBeLessThan(tokens.primary[1]); + expect(tokens.active[0]).toBeGreaterThan(tokens.primary[0]); +} + +async function expectSelectedSessionScopeTab(page: Page, accessibleName: string) { + const tablist = page.getByRole("tablist", { name: "Session scope" }); + const selected = tablist.getByRole("tab", { name: accessibleName, exact: true }); + const inactive = tablist.locator('[role="tab"][aria-selected="false"]'); + await expect(tablist.locator('[role="tab"][aria-selected="true"]')).toHaveCount(1); + await expect(selected).toHaveAttribute("data-tab-state", "active"); + expect(await selected.getAttribute("class")).toContain("bg-[oklch(var(--nav-active))]"); + + const [selectedStyle, inactiveStyle] = await Promise.all([ + selected.evaluate((element) => ({ + backgroundColor: getComputedStyle(element).backgroundColor, + borderColor: getComputedStyle(element).borderBottomColor, + })), + inactive.evaluate((element) => ({ + backgroundColor: getComputedStyle(element).backgroundColor, + borderColor: getComputedStyle(element).borderBottomColor, + borderWidth: getComputedStyle(element).borderTopWidth, + })), + ]); + expect(selectedStyle.backgroundColor).not.toBe(inactiveStyle.backgroundColor); + expect(selectedStyle.borderColor).not.toBe(inactiveStyle.borderColor); + expect(inactiveStyle.borderWidth).toBe("1px"); + expect(inactiveStyle.backgroundColor).not.toMatch(/^(?:transparent|rgba\(0, 0, 0, 0\))$/); + expect(inactiveStyle.borderColor).not.toMatch(/^(?:transparent|rgba\(0, 0, 0, 0\))$/); +} + +function cssLightness(color: string): number { + const oklab = color.match(/^okl(?:ab|ch)\(([\d.]+)(%)?/); + if (oklab) { + const value = Number(oklab[1]); + return oklab[2] ? value / 100 : value; + } + const channels = color.match(/[\d.]+/g)?.slice(0, 3).map(Number) ?? []; + if (channels.length !== 3) return Number.NaN; + return color.startsWith("color(srgb") ? Math.max(...channels) : Math.max(...channels) / 255; +} + +async function expectFleetRowTooltipBelow(trigger: Locator, expectedContent: string) { + await trigger.hover(); + await expect.poll(() => trigger.evaluate((element) => { + const owner = element.closest("[data-tooltip]"); + return owner ? getComputedStyle(owner, "::after").visibility : "missing"; + })).toBe("visible"); + await expect.poll(() => trigger.evaluate((element) => { + const owner = element.closest("[data-tooltip]"); + return owner ? getComputedStyle(owner, "::after").opacity : "missing"; + })).toBe("1"); + + const state = await trigger.evaluate((element) => { + const owner = element.closest("[data-tooltip]"); + if (!owner) throw new Error("Row action tooltip owner is missing"); + const style = getComputedStyle(owner, "::after"); + return { + content: style.content.replace(/^['\"]|['\"]$/g, ""), + ownerHeight: owner.getBoundingClientRect().height, + top: Number.parseFloat(style.top), + right: Number.parseFloat(style.right), + opacity: style.opacity, + pointerEvents: style.pointerEvents, + backgroundColor: style.backgroundColor, + color: style.color, + }; + }); + + expect(state.content).toBe(expectedContent); + expect(state.top - state.ownerHeight).toBeCloseTo(3, 1); + expect(state.right).toBe(0); + expect(state.opacity).toBe("1"); + expect(state.pointerEvents).toBe("none"); + expect(cssLightness(state.backgroundColor)).toBeLessThan(0.35); + expect(cssLightness(state.color)).toBeGreaterThan(0.9); +} + +async function expectContextPanelHeaderUsesPrimary( page: Page, accessibleName: string, ) { @@ -127,26 +255,22 @@ async function expectContextPanelHeaderMatchesNewSession( .getByRole("main", { name: "Database management" }); const panel = manager.getByRole("dialog", { name: accessibleName }); const header = panel.locator(':scope > [data-catalog-panel-region="header"]'); - const newSession = page - .getByRole("complementary", { name: "Session navigation" }) - .getByRole("button", { name: "New session", exact: true }); - await expect(header).toHaveCount(1); - await expect(newSession).toBeVisible(); - const [headerStyle, actionStyle] = await Promise.all([ + const currentNavigation = page.locator('[data-navigation-state="current"]'); + const [headerStyle, navigationStyle] = await Promise.all([ header.evaluate((element) => ({ backgroundColor: getComputedStyle(element).backgroundColor, backgroundImage: getComputedStyle(element).backgroundImage, })), - newSession.evaluate((element) => ({ + currentNavigation.evaluate((element) => ({ backgroundColor: getComputedStyle(element).backgroundColor, })), ]); expect(headerStyle.backgroundImage).toBe("none"); expect(headerStyle.backgroundColor).not.toBe("rgba(0, 0, 0, 0)"); - expect(headerStyle.backgroundColor).toBe(actionStyle.backgroundColor); + expect(headerStyle.backgroundColor).not.toBe(navigationStyle.backgroundColor); } async function expectWorkAreaPanelGeometry(page: Page, accessibleName: string) { @@ -181,7 +305,7 @@ async function expectWorkAreaPanelGeometry(page: Page, accessibleName: string) { expect(panelBox.y + panelBox.height).toBeLessThanOrEqual(workAreaBox.y + workAreaBox.height + 1); } -async function expectWorkAreaPanelHeaderMatchesNewSession( +async function expectWorkAreaPanelHeaderUsesPrimary( page: Page, accessibleName: string, ) { @@ -189,21 +313,19 @@ async function expectWorkAreaPanelHeaderMatchesNewSession( .getByTestId("conversation-column") .getByRole("dialog", { name: accessibleName }); const header = panel.locator(':scope > [data-work-area-panel-region="header"]'); - const newSession = page - .getByRole("complementary", { name: "Session navigation" }) - .getByRole("button", { name: "New session", exact: true }); - const [headerStyle, actionStyle] = await Promise.all([ + const currentNavigation = page.locator('[data-navigation-state="current"]'); + const [headerStyle, navigationStyle] = await Promise.all([ header.evaluate((element) => ({ backgroundColor: getComputedStyle(element).backgroundColor, backgroundImage: getComputedStyle(element).backgroundImage, })), - newSession.evaluate((element) => ({ + currentNavigation.evaluate((element) => ({ backgroundColor: getComputedStyle(element).backgroundColor, })), ]); expect(headerStyle.backgroundImage).toBe("none"); - expect(headerStyle.backgroundColor).toBe(actionStyle.backgroundColor); + expect(headerStyle.backgroundColor).not.toBe(navigationStyle.backgroundColor); } test.beforeAll(async () => { @@ -218,10 +340,14 @@ test.afterAll(async () => { test("context panels stay inside the manager and the Tables grid sits in a sidebar-colored frame", async ({ page }) => { const unexpectedCatalogRequests: string[] = []; const responses: Record = { - "GET /api/catalog/databases": [database], + "GET /api/catalog/databases": [database, unconfiguredDatabase], "GET /api/catalog/metadata-generation/models": { - models: [], - default: null, + models: [ + { id: "deepseek-v4-pro", label: "DeepSeek V4 Pro" }, + { id: "glm-53", label: "GLM 5.3" }, + { id: "qwen-36", label: "AritmoLab Qwen 3.6 35B A3B" }, + ], + default: "glm-53", }, "GET /api/catalog/description-generation-runs?limit=50": [], "GET /api/catalog/sensitive-data-suggestion-runs?limit=50": [], @@ -252,10 +378,40 @@ test("context panels stay inside the manager and the Tables grid sits in a sideb await page.goto(stack.publicUrl); await signInAsAdmin(page); + await expectCurrentNavigation(page, "New session"); + await expectSelectedSessionScopeTab(page, "My sessions"); + await page.getByRole("tab", { name: "All sessions", exact: true }).click(); + await expectSelectedSessionScopeTab(page, "All sessions"); + await page.getByRole("tab", { name: "My sessions", exact: true }).click(); + await expectSelectedSessionScopeTab(page, "My sessions"); - await page + const adminNavigationRail = page.getByRole("complementary", { name: "Session navigation" }); + const administration = adminNavigationRail.getByRole("button", { name: "Administration", exact: true }); + await expect(administration).toHaveAttribute("aria-expanded", "false"); + await expect(adminNavigationRail.getByRole("region", { name: "Administration" })).toHaveCount(0); + await administration.click(); + await expect(administration).toHaveAttribute("aria-expanded", "true"); + const administrationPanel = adminNavigationRail.getByRole("region", { name: "Administration" }); + await expect(administrationPanel).toBeVisible(); + expect(await administrationPanel.locator(":scope > *").evaluateAll((elements) => elements.map((element) => ( + element.getAttribute("role") === "separator" ? "separator" : element.textContent?.trim() + )))).toEqual([ + "Database management", + "separator", + "Workspace management", + "Pi management", + ]); + + await administration.click(); + await expect(administration).toHaveAttribute("aria-expanded", "false"); + await expect(adminNavigationRail.getByRole("button", { name: "Database management", exact: true })).toHaveCount(0); + await administration.click(); + await expect(administration).toHaveAttribute("aria-expanded", "true"); + + await adminNavigationRail .getByRole("button", { name: "Database management", exact: true }) .click(); + await expectCurrentNavigation(page, "Database management"); await expect( page.getByRole("complementary", { name: "Session navigation" }), @@ -268,27 +424,41 @@ test("context panels stay inside the manager and the Tables grid sits in a sideb await expect( page.getByRole("button", { name: "Back to workspace", exact: true }), ).toHaveCount(0); + await expect(page.getByRole("button", { name: /Add database/i })).toHaveCount(0); + await expect(page.getByRole("columnheader", { name: /Revision \/ Evidence/ })).toBeVisible(); + await expect(page.getByRole("columnheader", { name: /NL→SQL runtime/ })).toBeVisible(); + await expect(page.getByRole("columnheader", { name: /Metadata Catalog/ })).toBeVisible(); + const metadataModelSelector = page.getByRole("combobox", { + name: "Metadata-generation LLM model", + }); + await expect(metadataModelSelector).toHaveCount(1); + await expect(metadataModelSelector).toHaveValue("glm-53"); + await expect(metadataModelSelector.locator("option")).toHaveText([ + "DeepSeek V4 Pro", + "GLM 5.3", + "AritmoLab Qwen 3.6 35B A3B", + ]); + + await page + .getByRole("button", { name: "Configure catalog for Research laboratory", exact: true }) + .click(); + const configurePanel = page.getByRole("dialog", { name: "Configure catalog" }); + await expectContextPanelGeometry(page, "Configure catalog"); + const configuredWorkspace = configurePanel.getByLabel("Workspace", { exact: true }); + await expect(configuredWorkspace).toHaveValue("Research laboratory"); + await expect(configuredWorkspace).toHaveAttribute("readonly", ""); + await configurePanel.getByRole("button", { name: "Close database panel" }).click(); await page .getByRole("button", { name: "Edit Policlinico San Donato", exact: true }) .click(); await expectContextPanelGeometry(page, "Edit database"); - await expectContextPanelHeaderMatchesNewSession(page, "Edit database"); + await expectContextPanelHeaderUsesPrimary(page, "Edit database"); await page .getByRole("dialog", { name: "Edit database" }) .getByRole("button", { name: "Close database panel" }) .click(); - await page - .getByRole("button", { name: "Description history", exact: true }) - .click(); - await expectContextPanelGeometry(page, "Description generation"); - await expectContextPanelHeaderMatchesNewSession(page, "Description generation"); - await page - .getByRole("dialog", { name: "Description generation" }) - .getByRole("button", { name: /close/i }) - .click(); - await page.setViewportSize({ width: 1280, height: 800 }); await page .getByRole("button", { name: "Edit Policlinico San Donato", exact: true }) @@ -300,12 +470,12 @@ test("context panels stay inside the manager and the Tables grid sits in a sideb .click(); await page.setViewportSize({ width: 1910, height: 911 }); - await page - .getByRole("button", { - name: "View tables for Policlinico San Donato", - exact: true, - }) - .click(); + const databaseTablesAction = page.getByRole("button", { + name: "View tables for Policlinico San Donato", + exact: true, + }); + await expectFleetRowTooltipBelow(databaseTablesAction, "Tables"); + await databaseTablesAction.click(); await expect( page.getByRole("region", { @@ -313,6 +483,16 @@ test("context panels stay inside the manager and the Tables grid sits in a sideb }), ).toBeVisible(); + await page + .getByRole("button", { name: "Description history", exact: true }) + .click(); + await expectContextPanelGeometry(page, "Description generation"); + await expectContextPanelHeaderUsesPrimary(page, "Description generation"); + await page + .getByRole("dialog", { name: "Description generation" }) + .getByRole("button", { name: /close/i }) + .click(); + const applicationBar = page.locator( 'main[aria-label="Database management"] .thot-fleet-ledger__header', ); @@ -326,6 +506,10 @@ test("context panels stay inside the manager and the Tables grid sits in a sideb await expect( applicationBar.getByRole("group", { name: "Database management actions" }), ).toBeVisible(); + await expect( + applicationBar.getByRole("combobox", { name: "Metadata-generation LLM model" }), + ).toHaveValue("glm-53"); + await expect(metadataModelSelector).toHaveCount(1); await expect(applicationBar).not.toContainText("Thoth catalog · Fleet ledger"); await expect(applicationBar).not.toContainText("Inspect physical metadata"); @@ -355,11 +539,14 @@ test("context panels stay inside the manager and the Tables grid sits in a sideb await expect(page.getByRole("row", { name: /patients/ })).toBeVisible(); - await page - .getByRole("button", { name: "Edit description for patients", exact: true }) - .click(); + const editPatientsAction = page.getByRole("button", { + name: "Edit description for patients", + exact: true, + }); + await expectFleetRowTooltipBelow(editPatientsAction, "Edit metadata"); + await editPatientsAction.click(); await expectContextPanelGeometry(page, "Review table description"); - await expectContextPanelHeaderMatchesNewSession(page, "Review table description"); + await expectContextPanelHeaderUsesPrimary(page, "Review table description"); const tableEditor = page.getByRole("dialog", { name: "Review table description" }); const tableEditorFooter = tableEditor.locator( ':scope > [data-catalog-panel-region="footer"]', @@ -500,6 +687,10 @@ test("Workspace and Pi management share the centered work-area panel without cov await page.goto(stack.publicUrl); await signInAsAdmin(page); + const administration = page.getByRole("button", { name: "Administration", exact: true }); + await expect(administration).toHaveAttribute("aria-expanded", "false"); + await administration.click(); + await expect(administration).toHaveAttribute("aria-expanded", "true"); for (const viewport of [ { width: 1910, height: 911 }, @@ -511,25 +702,29 @@ test("Workspace and Pi management share the centered work-area panel without cov exact: true, }); await workspaceTrigger.click(); + await expectCurrentNavigation(page, "Workspace management"); await expectWorkAreaPanelGeometry(page, "Workspace management"); - await expectWorkAreaPanelHeaderMatchesNewSession(page, "Workspace management"); + await expectWorkAreaPanelHeaderUsesPrimary(page, "Workspace management"); const piTrigger = page.getByRole("button", { name: "Pi management", exact: true }); await piTrigger.click(); + await expectCurrentNavigation(page, "Pi management"); await expect( page.getByRole("dialog", { name: "Workspace management" }), ).toHaveCount(0); await expectWorkAreaPanelGeometry(page, "Pi management"); - await expectWorkAreaPanelHeaderMatchesNewSession(page, "Pi management"); + await expectWorkAreaPanelHeaderUsesPrimary(page, "Pi management"); await page .getByRole("dialog", { name: "Pi management" }) .getByRole("button", { name: "Close Pi management" }) .click(); await expect(piTrigger).toBeFocused(); + await expectCurrentNavigation(page, "New session"); } await page.setViewportSize({ width: 720, height: 800 }); await page.getByRole("button", { name: "Workspace management", exact: true }).click(); + await expectCurrentNavigation(page, "Workspace management"); const compactWorkArea = page.getByTestId("conversation-column"); const compactPanel = compactWorkArea.getByRole("dialog", { name: "Workspace management" }); const [compactWorkAreaBox, compactPanelBox] = await Promise.all([ diff --git a/frontend/src/api/catalog-databases.test.ts b/frontend/src/api/catalog-databases.test.ts index dd59c07c..578187de 100644 --- a/frontend/src/api/catalog-databases.test.ts +++ b/frontend/src/api/catalog-databases.test.ts @@ -1,7 +1,15 @@ import { http, HttpResponse } from "msw"; import { expect, test } from "vitest"; import { server } from "../test/msw"; -import { getCatalogMetrics, type CatalogMetrics } from "./catalog-databases"; +import { + createCatalogRelationship, + deleteCatalogRelationship, + getCatalogMetrics, + rebuildGeneratedRelationships, + setCatalogRelationshipStatus, + type CatalogMetrics, + type CatalogRelationship, +} from "./catalog-databases"; const databaseId = "d4baf0f5-b9c3-4dc5-aad2-2f5676a36e64"; @@ -58,3 +66,99 @@ test("propagates an unsuccessful catalog metrics response", async () => { await expect(getCatalogMetrics()).rejects.toMatchObject({ status: 503 }); }); + +const logicalRelationship: CatalogRelationship = { + id: "11111111-1111-4111-8111-111111111111", + databaseId, + constraintName: null, + sourceTableId: "22222222-2222-4222-8222-222222222222", + sourceTableName: "orders", + targetTableId: "33333333-3333-4333-8333-333333333333", + targetTableName: "users", + updateRule: null, + deleteRule: null, + deferrable: false, + initiallyDeferred: false, + columns: [{ + position: 1, + sourceColumnId: "44444444-4444-4444-8444-444444444444", + sourceColumnName: "user_id", + targetColumnId: "55555555-5555-4555-8555-555555555555", + targetColumnName: "id", + }], + origin: "manual", + status: "active", + lastSyncedDatabaseVersion: null, + lastSyncedAt: null, + createdAt: "2026-08-31T12:00:00.000Z", + updatedAt: "2026-08-31T12:00:00.000Z", +}; + +test("creates a manual relationship from the selected source and target columns", async () => { + let requestBody: unknown; + server.use(http.post("/api/catalog/databases/:databaseId/relationships", async ({ request }) => { + requestBody = await request.json(); + return HttpResponse.json(logicalRelationship, { status: 201 }); + })); + + await expect(createCatalogRelationship( + databaseId, + logicalRelationship.columns[0].sourceColumnId, + logicalRelationship.columns[0].targetColumnId, + )).resolves.toEqual(logicalRelationship); + expect(requestBody).toEqual({ + sourceColumnId: logicalRelationship.columns[0].sourceColumnId, + targetColumnId: logicalRelationship.columns[0].targetColumnId, + }); +}); + +test("rebuilds generated relationships for one database", async () => { + let calls = 0; + server.use(http.post( + "/api/catalog/databases/:databaseId/relationships/rebuild-generated", + () => { + calls += 1; + return HttpResponse.json({ added: 12, alreadyPresent: 7, excluded: 3, ambiguous: 2 }); + }, + )); + + await expect(rebuildGeneratedRelationships(databaseId)).resolves.toEqual({ + added: 12, + alreadyPresent: 7, + excluded: 3, + ambiguous: 2, + }); + expect(calls).toBe(1); +}); + +test("changes whether a logical relationship is active or excluded", async () => { + let requestBody: unknown; + server.use(http.patch( + "/api/catalog/databases/:databaseId/relationships/:relationshipId", + async ({ request }) => { + requestBody = await request.json(); + return HttpResponse.json({ ...logicalRelationship, status: "excluded" }); + }, + )); + + await expect(setCatalogRelationshipStatus( + databaseId, + logicalRelationship.id, + "excluded", + )).resolves.toMatchObject({ status: "excluded" }); + expect(requestBody).toEqual({ status: "excluded" }); +}); + +test("permanently deletes a logical relationship", async () => { + let calls = 0; + server.use(http.delete( + "/api/catalog/databases/:databaseId/relationships/:relationshipId", + () => { + calls += 1; + return new HttpResponse(null, { status: 204 }); + }, + )); + + await expect(deleteCatalogRelationship(databaseId, logicalRelationship.id)).resolves.toBeUndefined(); + expect(calls).toBe(1); +}); diff --git a/frontend/src/api/catalog-databases.ts b/frontend/src/api/catalog-databases.ts index aa0e087b..6bac5f5c 100644 --- a/frontend/src/api/catalog-databases.ts +++ b/frontend/src/api/catalog-databases.ts @@ -3,6 +3,12 @@ import { joinBackendPath } from "./runtime-config"; export type DatabaseTransport = "postgres_direct" | "rest_api" | "ssh_tunnel"; export type ConnectionStatus = "untested" | "reachable" | "failed"; +export type WorkspaceEvidenceState = + | "not_declared" + | "materialized_current_revision" + | "configuration_required" + | "configured_unverified" + | "workspace_unavailable"; export type CatalogSecretName = | "password" | "apiKey" @@ -33,6 +39,16 @@ export interface CatalogDatabase { workspaceName: string; workspaceDescription?: string; workspaceAvailable: boolean; + workspaceRevision: { commit: string; blob: string } | null; + workspaceEvidence: { + sourceType: "filesystem" | "http" | "s3" | null; + state: WorkspaceEvidenceState; + }; + runtimeBinding: { + transport: DatabaseTransport; + configurationState: "ready" | "configuration_required"; + sessionTransportSupported: boolean; + } | null; configured: boolean; engine: "postgres"; databaseName: string; @@ -164,25 +180,37 @@ export interface CatalogRelationshipColumn { targetColumnName: string; } +export type CatalogRelationshipOrigin = "physical" | "generated" | "manual"; +export type CatalogRelationshipStatus = "active" | "excluded"; + export interface CatalogRelationship { id: string; databaseId: string; - constraintName: string; + constraintName: string | null; sourceTableId: string; sourceTableName: string; targetTableId: string; targetTableName: string; - updateRule: string; - deleteRule: string; + updateRule: string | null; + deleteRule: string | null; deferrable: boolean; initiallyDeferred: boolean; columns: CatalogRelationshipColumn[]; + origin: CatalogRelationshipOrigin; + status: CatalogRelationshipStatus; lastSyncedDatabaseVersion: number | null; lastSyncedAt: string | null; createdAt: string; updatedAt: string; } +export interface CatalogRelationshipRebuildResult { + added: number; + alreadyPresent: number; + excluded: number; + ambiguous: number; +} + export type CatalogDatabaseMetadataDeleteTarget = "tables" | "relationships"; export type CatalogTableMetadataDeleteTarget = "columns" | "relationships"; @@ -452,6 +480,36 @@ export const listSensitiveDataSuggestionEvents = (runId: string, after = 0) => export const listCatalogRelationships = (databaseId: string) => apiFetch(`/catalog/databases/${encodeURIComponent(databaseId)}/relationships`); +export const createCatalogRelationship = ( + databaseId: string, + sourceColumnId: string, + targetColumnId: string, +) => apiFetch( + `/catalog/databases/${encodeURIComponent(databaseId)}/relationships`, + { method: "POST", body: JSON.stringify({ sourceColumnId, targetColumnId }) }, +); + +export const rebuildGeneratedRelationships = (databaseId: string) => + apiFetch( + `/catalog/databases/${encodeURIComponent(databaseId)}/relationships/rebuild-generated`, + { method: "POST" }, + ); + +export const setCatalogRelationshipStatus = ( + databaseId: string, + relationshipId: string, + status: CatalogRelationshipStatus, +) => apiFetch( + `/catalog/databases/${encodeURIComponent(databaseId)}/relationships/${encodeURIComponent(relationshipId)}`, + { method: "PATCH", body: JSON.stringify({ status }) }, +); + +export const deleteCatalogRelationship = (databaseId: string, relationshipId: string) => + apiFetch( + `/catalog/databases/${encodeURIComponent(databaseId)}/relationships/${encodeURIComponent(relationshipId)}`, + { method: "DELETE" }, + ); + export const deleteCatalogDatabaseMetadata = ( databaseIds: string[], target: CatalogDatabaseMetadataDeleteTarget, diff --git a/frontend/src/api/client.test.ts b/frontend/src/api/client.test.ts index 0d50a9bc..dfbe069e 100644 --- a/frontend/src/api/client.test.ts +++ b/frontend/src/api/client.test.ts @@ -152,6 +152,15 @@ test.each([ ["sensitive_data_suggestion_history_request_invalid", "Sensitive suggestion history parameters are invalid."], ["sensitive_data_suggestion_history_failed", "Sensitive suggestion history could not be loaded."], ["sensitive_data_suggestion_run_not_found", "The sensitive suggestion run was not found."], + ["relationship_not_found", "The relationship no longer exists. Refresh and try again."], + ["relationship_duplicate", "This relationship already exists."], + ["relationship_target_not_unique", "The target column must be the only primary-key column of its table."], + ["relationship_type_incompatible", "Source and target column types are not compatible."], + ["relationship_read_only", "Physical relationships are read-only."], + ["relationship_request_invalid", "The relationship request is invalid."], + ["relationship_operation_failed", "The relationship operation failed."], + ["relationship_schema_stale", "Synchronize the current database schema before managing logical relationships."], + ["column_not_found", "The selected catalog column was not found. Refresh and try again."], ])("maps the catalog error code %s to safe local copy", async (code, message) => { const fetchSpy = vi.spyOn(globalThis, "fetch").mockResolvedValue( new Response(JSON.stringify({ code, message: "provider detail must not be trusted" }), { diff --git a/frontend/src/api/client.ts b/frontend/src/api/client.ts index 5cbcebf3..7c1bd75c 100644 --- a/frontend/src/api/client.ts +++ b/frontend/src/api/client.ts @@ -35,6 +35,9 @@ const safeErrorCodes = new Set([ "sensitive_data_suggestion_run_not_found", "schema_sync_conflict", "schema_introspection_failed", "schema_request_invalid", "schema_operation_failed", "sync_run_not_found", "table_stale", "column_stale", + "relationship_not_found", "relationship_duplicate", "relationship_target_not_unique", + "relationship_type_incompatible", "relationship_read_only", "relationship_request_invalid", + "relationship_operation_failed", "relationship_schema_stale", "column_not_found", ]); type SafeErrorPayload = { @@ -104,6 +107,15 @@ const localCodeMessages: Record = { sync_run_not_found: "The synchronization run was not found.", table_stale: "Table metadata changed. Reload and try again.", column_stale: "Column metadata changed. Reload and try again.", + relationship_not_found: "The relationship no longer exists. Refresh and try again.", + relationship_duplicate: "This relationship already exists.", + relationship_target_not_unique: "The target column must be the only primary-key column of its table.", + relationship_type_incompatible: "Source and target column types are not compatible.", + relationship_read_only: "Physical relationships are read-only.", + relationship_request_invalid: "The relationship request is invalid.", + relationship_operation_failed: "The relationship operation failed.", + relationship_schema_stale: "Synchronize the current database schema before managing logical relationships.", + column_not_found: "The selected catalog column was not found. Refresh and try again.", }; const localStatusMessages: Record = { diff --git a/frontend/src/components/ui/button.tsx b/frontend/src/components/ui/button.tsx index dd37ba3a..5894f7c8 100644 --- a/frontend/src/components/ui/button.tsx +++ b/frontend/src/components/ui/button.tsx @@ -13,6 +13,8 @@ const buttonVariants = cva( "border-border bg-card shadow-xs hover:bg-muted hover:text-foreground aria-expanded:bg-muted aria-expanded:text-foreground dark:border-input dark:bg-input/30 dark:hover:bg-input/50", secondary: "bg-secondary text-secondary-foreground hover:bg-[color-mix(in_oklch,oklch(var(--secondary)),oklch(var(--foreground))_5%)] aria-expanded:bg-secondary aria-expanded:text-secondary-foreground", + navigationActive: + "border-[oklch(var(--nav-active-border))] bg-[oklch(var(--nav-active))] text-[oklch(var(--nav-active-foreground))] shadow-xs hover:bg-[oklch(var(--nav-active-hover))] hover:text-[oklch(var(--nav-active-foreground))] focus-visible:border-[oklch(var(--nav-active-border))] focus-visible:ring-[oklch(var(--nav-active-border)/0.32)]", ghost: "hover:bg-muted hover:text-foreground aria-expanded:bg-muted aria-expanded:text-foreground dark:hover:bg-muted/50", destructive: diff --git a/frontend/src/index.css b/frontend/src/index.css index 86bad42c..d85ee0b2 100644 --- a/frontend/src/index.css +++ b/frontend/src/index.css @@ -47,6 +47,12 @@ --success: 0.7577 0.1581 165.0; --warning: 0.8523 0.1386 78.9; --info: 0.7035 0.1128 221.3; + /* Current navigation shares Instrument Red's hue, with lower chroma and + higher lightness so selection reads as a muted location signal. */ + --nav-active: 0.9250 0.0520 23.2; + --nav-active-hover: 0.8950 0.0710 23.2; + --nav-active-foreground: 0.3650 0.1100 23.2; + --nav-active-border: 0.6000 0.1350 23.2; --radius: 0.5rem; --sidebar: 0.9709 0.0011 17.2; --sidebar-foreground: 0.2678 0.0097 355.6; @@ -81,6 +87,10 @@ --border: 0.3715 0 90; --input: 0.3715 0 90; --ring: 0.6897 0.1779 16.9; + --nav-active: 0.3350 0.0550 20.3; + --nav-active-hover: 0.3750 0.0650 20.3; + --nav-active-foreground: 0.8900 0.0700 20.3; + --nav-active-border: 0.6800 0.1350 20.3; --sidebar: 0.2350 0 90; --sidebar-foreground: 0.9310 0 90; --sidebar-primary: 0.6023 0.1848 20.3; diff --git a/frontend/src/shell/AppShell.auth.test.tsx b/frontend/src/shell/AppShell.auth.test.tsx index 7146b812..650de459 100644 --- a/frontend/src/shell/AppShell.auth.test.tsx +++ b/frontend/src/shell/AppShell.auth.test.tsx @@ -1,4 +1,4 @@ -import { act, render, screen, waitFor } from "@testing-library/react"; +import { act, render, screen, waitFor, within } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; import { http, HttpResponse } from "msw"; @@ -45,24 +45,50 @@ beforeEach(() => { }); describe("authenticated shell permissions", () => { - test("shows only read-safe workspace chrome to a session user", async () => { + test("hides administrative navigation from a session user", async () => { renderShell({ subject: "user-1", isAdmin: false, roles: ["user"], permissions: ["session.use"] }); - expect(await screen.findByRole("button", { name: "Workspace management" })).toBeInTheDocument(); + expect(await screen.findByRole("button", { name: "New session" })).toBeInTheDocument(); + expect(screen.queryByRole("button", { name: "Administration" })).not.toBeInTheDocument(); + expect(screen.queryByRole("button", { name: "Database management" })).not.toBeInTheDocument(); + expect(screen.queryByRole("button", { name: "Workspace management" })).not.toBeInTheDocument(); expect(screen.queryByRole("button", { name: "Pi management" })).not.toBeInTheDocument(); - expect(screen.queryByRole("button", { name: "All sessions" })).not.toBeInTheDocument(); + expect(screen.queryByRole("tab", { name: "All sessions" })).not.toBeInTheDocument(); }); - test("shows management and all-session chrome only for exact permissions", async () => { + test("shows the ordered administrative accordion only to an admin", async () => { + const user = userEvent.setup(); renderShell({ subject: "admin-1", isAdmin: true, roles: ["admin"], - permissions: ["session.use", "session.read_all", "workspace.manage", "workspace.secrets.manage", "pi.manage"], + permissions: ["session.use", "session.read_all", "workspace.manage", "workspace.secrets.manage", "database.manage", "pi.manage"], }); - expect(await screen.findByRole("button", { name: "Pi management" })).toBeInTheDocument(); - expect(screen.getByRole("button", { name: "All sessions" })).toBeInTheDocument(); + const sessionNavigation = screen.getByRole("complementary", { name: "Session navigation" }); + const trigger = await within(sessionNavigation).findByRole("button", { name: "Administration" }); + expect(trigger).toHaveAttribute("aria-expanded", "false"); + expect(within(sessionNavigation).queryByRole("region", { name: "Administration" })).not.toBeInTheDocument(); + + trigger.focus(); + await user.keyboard("{Enter}"); + expect(trigger).toHaveAttribute("aria-expanded", "true"); + + const panel = within(sessionNavigation).getByRole("region", { name: "Administration" }); + const database = within(panel).getByRole("button", { name: "Database management" }); + const separator = within(panel).getByRole("separator"); + const workspace = within(panel).getByRole("button", { name: "Workspace management" }); + const pi = within(panel).getByRole("button", { name: "Pi management" }); + expect(panel.children[0]).toBe(database); + expect(panel.children[1]).toBe(separator); + expect(panel.children[2]).toBe(workspace); + expect(panel.children[3]).toBe(pi); + expect(screen.getByRole("tab", { name: "All sessions" })).toBeInTheDocument(); + + await user.keyboard(" "); + expect(trigger).toHaveAttribute("aria-expanded", "false"); + expect(within(sessionNavigation).queryByRole("region", { name: "Administration" })).not.toBeInTheDocument(); + expect(within(sessionNavigation).queryByRole("button", { name: "Database management" })).not.toBeInTheDocument(); }); test("keeps identity but hides logout outside local authentication", async () => { @@ -178,15 +204,14 @@ describe("authenticated shell permissions", () => { subject: "admin-1", isAdmin: true, roles: ["admin"], permissions: ["session.use", "session.read_all", "workspace.manage", "pi.manage"], }); - expect(await screen.findByRole("button", { name: "Pi management" })).toBeInTheDocument(); - expect(screen.getByRole("button", { name: "All sessions" })).toBeInTheDocument(); + expect(await screen.findByRole("button", { name: "Administration" })).toBeInTheDocument(); + expect(screen.getByRole("tab", { name: "All sessions" })).toBeInTheDocument(); act(() => clearAuthState()); + expect(screen.queryByRole("button", { name: "Administration" })).not.toBeInTheDocument(); + expect(screen.queryByRole("button", { name: "Workspace management" })).not.toBeInTheDocument(); expect(screen.queryByRole("button", { name: "Pi management" })).not.toBeInTheDocument(); - expect(screen.queryByRole("button", { name: "All sessions" })).not.toBeInTheDocument(); - await userEvent.click(screen.getByRole("button", { name: "Workspace management" })); - expect(await screen.findByRole("heading", { name: "Workspace management" })).toBeInTheDocument(); - expect(screen.queryByRole("button", { name: "Update workspace repository" })).not.toBeInTheDocument(); + expect(screen.queryByRole("tab", { name: "All sessions" })).not.toBeInTheDocument(); }); test("remounts the real shell so user-A panel and transcript state cannot survive user-B", async () => { diff --git a/frontend/src/shell/AppShell.database-management.test.tsx b/frontend/src/shell/AppShell.database-management.test.tsx index c5dc4047..d5939a89 100644 --- a/frontend/src/shell/AppShell.database-management.test.tsx +++ b/frontend/src/shell/AppShell.database-management.test.tsx @@ -17,6 +17,10 @@ function renderShell() { ); } +async function expandAdministration() { + await userEvent.click(screen.getByRole("button", { name: "Administration" })); +} + beforeEach(() => { clearAuthState(); setAuthState({ @@ -92,10 +96,19 @@ test("uses Fleet Ledger by default and gates the legacy fallback to non-producti test("replaces the core conversation while keeping the session navigation", async () => { renderShell(); + await expandAdministration(); const conversationColumn = screen.getByTestId("conversation-column"); const sessionNavigation = screen.getByRole("complementary", { name: "Session navigation" }); + const newSession = within(sessionNavigation).getByRole("button", { name: "New session" }); const databaseManagement = within(sessionNavigation).getByRole("button", { name: "Database management" }); + + expect(newSession).toHaveAttribute("aria-current", "page"); + expect(newSession).toHaveAttribute("data-navigation-state", "current"); + expect(newSession).toHaveClass("bg-[oklch(var(--nav-active))]"); + expect(databaseManagement).toHaveAttribute("data-navigation-state", "available"); + expect(databaseManagement).not.toHaveAttribute("aria-current"); + await userEvent.click(databaseManagement); const manager = screen.getByRole("main", { name: "Database management" }); @@ -103,8 +116,13 @@ test("replaces the core conversation while keeping the session navigation", asyn expect(conversationColumn).toContainElement(manager); expect(screen.getByRole("complementary", { name: "Session navigation" })).toBe(sessionNavigation); expect(sessionNavigation).toBeVisible(); - expect(within(sessionNavigation).getByRole("button", { name: "New session" })).toBeVisible(); + expect(newSession).toBeVisible(); + expect(newSession).toHaveAttribute("data-navigation-state", "available"); + expect(newSession).not.toHaveAttribute("aria-current"); + expect(newSession).not.toHaveClass("bg-[oklch(var(--nav-active))]"); expect(databaseManagement).toHaveAttribute("aria-current", "page"); + expect(databaseManagement).toHaveAttribute("data-navigation-state", "current"); + expect(databaseManagement).toHaveClass("bg-[oklch(var(--nav-active))]"); expect(screen.queryByRole("button", { name: "Back to workspace" })).not.toBeInTheDocument(); expect(screen.queryByRole("textbox", { name: /new question/i })).not.toBeInTheDocument(); }); @@ -135,6 +153,7 @@ test("keeps a live core session connected while returning from database manageme })), ); renderShell(); + await expandAdministration(); const session = await screen.findByTestId("session-item-s1"); await userEvent.click(session); @@ -157,7 +176,7 @@ test("keeps a live core session connected while returning from database manageme expect(FakeEventSource.instances).toHaveLength(1); }); -test("keeps read-safe workspace access but hides privileged management entries", () => { +test("hides the complete administrative navigation from a regular user", () => { clearAuthState(); setAuthState({ issuer: "test", @@ -171,7 +190,9 @@ test("keeps read-safe workspace access but hides privileged management entries", renderShell(); - expect(screen.getByRole("button", { name: "Workspace management" })).toBeInTheDocument(); + expect(screen.getByRole("button", { name: "New session" })).toBeInTheDocument(); + expect(screen.queryByRole("button", { name: "Administration" })).not.toBeInTheDocument(); + expect(screen.queryByRole("button", { name: "Workspace management" })).not.toBeInTheDocument(); expect(screen.queryByRole("button", { name: "Database management" })).not.toBeInTheDocument(); expect(screen.queryByRole("button", { name: "Pi management" })).not.toBeInTheDocument(); }); @@ -182,6 +203,13 @@ test("does not leave database management without confirming a dirty form", async workspaceId: "psd-clinical", workspaceName: "Policlinico San Donato", workspaceAvailable: true, + workspaceRevision: { commit: "a".repeat(40), blob: "b".repeat(40) }, + workspaceEvidence: { sourceType: "filesystem", state: "materialized_current_revision" }, + runtimeBinding: { + transport: "postgres_direct", + configurationState: "ready", + sessionTransportSupported: true, + }, configured: true, engine: "postgres", databaseName: "warehouse", @@ -202,6 +230,7 @@ test("does not leave database management without confirming a dirty form", async }]))); const confirm = vi.spyOn(window, "confirm").mockReturnValue(false); renderShell(); + await expandAdministration(); await userEvent.click(screen.getByRole("button", { name: "Database management" })); await userEvent.click(await screen.findByRole("button", { name: "Edit Policlinico San Donato" })); diff --git a/frontend/src/shell/AppShell.new-session.test.tsx b/frontend/src/shell/AppShell.new-session.test.tsx index 9a75d9a3..32cecfbc 100644 --- a/frontend/src/shell/AppShell.new-session.test.tsx +++ b/frontend/src/shell/AppShell.new-session.test.tsx @@ -156,6 +156,11 @@ test("model selector shows the three Pi-enabled models and stores the selected p }); test("opens Workspace management from the right sidebar without interrupting the shell", async () => { + setAuthState({ + issuer: "test", subject: "admin", roles: ["admin"], + permissions: ["session.use", "workspace.manage", "workspace.secrets.manage", "database.manage", "pi.manage"], + isAdmin: true, csrfToken: null, session: null, + }); server.use( http.get("/api/workspace-registry/status", () => HttpResponse.json({ branch: "main", ahead: 0, behind: 0, degraded: false })), @@ -163,15 +168,31 @@ test("opens Workspace management from the right sidebar without interrupting the ); renderShell(); + await userEvent.click(screen.getByRole("button", { name: "Administration" })); await userEvent.click(screen.getByRole("button", { name: "Workspace management" })); expect(await screen.findByRole("heading", { name: "Workspace management" })).toBeVisible(); const dialog = screen.getByRole("dialog", { name: "Workspace management" }); const workArea = screen.getByTestId("conversation-column"); const sessionNavigation = screen.getByRole("complementary", { name: "Session navigation" }); + const workspaceManagement = within(sessionNavigation).getByRole("button", { name: "Workspace management" }); + const newSession = within(sessionNavigation).getByRole("button", { name: "New session" }); expect(workArea).toContainElement(dialog); expect(sessionNavigation).not.toContainElement(dialog); expect(screen.getByTestId("app-shell")).toHaveAttribute("data-activity-layout", "closed"); + expect(workspaceManagement).toHaveAttribute("aria-current", "page"); + expect(workspaceManagement).toHaveAttribute("aria-expanded", "true"); + expect(workspaceManagement).toHaveAttribute("data-navigation-state", "current"); + expect(workspaceManagement).toHaveClass("bg-[oklch(var(--nav-active))]"); + expect(newSession).toHaveAttribute("data-navigation-state", "available"); + expect(newSession).not.toHaveAttribute("aria-current"); + + await userEvent.click(screen.getByRole("button", { name: "Close workspace management" })); + await waitFor(() => expect(screen.queryByRole("dialog", { name: "Workspace management" })).not.toBeInTheDocument()); + expect(newSession).toHaveAttribute("aria-current", "page"); + expect(newSession).toHaveAttribute("data-navigation-state", "current"); + expect(workspaceManagement).toHaveAttribute("aria-expanded", "false"); + expect(workspaceManagement).toHaveAttribute("data-navigation-state", "available"); }); test("does not block the shell when the DWH is unavailable at startup", async () => { @@ -184,7 +205,8 @@ test("does not block the shell when the DWH is unavailable at startup", async () ); renderShell(); - expect(await screen.findByRole("button", { name: "Workspace management" })).toBeVisible(); + expect(await screen.findByRole("button", { name: "New session" })).toBeVisible(); + expect(screen.queryByRole("button", { name: "Administration" })).not.toBeInTheDocument(); expect(screen.queryByRole("heading", { name: "Connection unavailable" })).not.toBeInTheDocument(); expect(healthChecks).toBe(0); }); diff --git a/frontend/src/shell/AppShell.session-mgmt.test.tsx b/frontend/src/shell/AppShell.session-mgmt.test.tsx index cc08a04d..243eaac4 100644 --- a/frontend/src/shell/AppShell.session-mgmt.test.tsx +++ b/frontend/src/shell/AppShell.session-mgmt.test.tsx @@ -16,7 +16,7 @@ const regularUser: AuthenticatedUser = { }; const adminUser: AuthenticatedUser = { ...regularUser, subject: "alice-id", roles: ["admin"] as const, - permissions: ["session.use", "session.read_all", "pi.manage"] as const, isAdmin: true, + permissions: ["session.use", "session.read_all", "workspace.manage", "workspace.secrets.manage", "database.manage", "pi.manage"] as const, isAdmin: true, }; function wrap(user: AuthenticatedUser = regularUser) { @@ -95,7 +95,8 @@ test("regular users load only their sessions and never see administrator control wrap(); await screen.findByText("Attiva uno"); expect(scope).toBe("mine"); - expect(screen.queryByRole("button", { name: "All sessions" })).not.toBeInTheDocument(); + expect(screen.queryByRole("button", { name: "Administration" })).not.toBeInTheDocument(); + expect(screen.queryByRole("tab", { name: "All sessions" })).not.toBeInTheDocument(); expect(screen.queryByText(/administrator view/i)).not.toBeInTheDocument(); }); @@ -115,17 +116,25 @@ test("Pi management preserves the open session summary and the model activity ti http.post("/api/sessions/:id/resume", () => resumeResult("s1")), http.get("/api/sessions/:id", () => HttpResponse.json({ phase: 4 })), ); - wrap(); + wrap(adminUser); await user.click(await screen.findByText("Attiva uno")); expect(await screen.findByRole("complementary", { name: "Session summary" })).toHaveTextContent("Domanda originale"); + await user.click(screen.getByRole("button", { name: "Administration" })); await user.click(screen.getByRole("button", { name: "Pi management" })); expect(await screen.findByRole("heading", { name: "Pi management" })).toBeVisible(); const dialog = screen.getByRole("dialog", { name: "Pi management" }); const workArea = screen.getByTestId("conversation-column"); const sessionNavigation = screen.getByRole("complementary", { name: "Session navigation" }); + const piManagement = within(sessionNavigation).getByRole("button", { name: "Pi management" }); + const newSession = within(sessionNavigation).getByRole("button", { name: "New session" }); expect(workArea).toContainElement(dialog); expect(sessionNavigation).not.toContainElement(dialog); + expect(piManagement).toHaveAttribute("aria-current", "page"); + expect(piManagement).toHaveAttribute("aria-expanded", "true"); + expect(piManagement).toHaveAttribute("data-navigation-state", "current"); + expect(newSession).toHaveAttribute("data-navigation-state", "available"); + expect(newSession).not.toHaveAttribute("aria-current"); const preservedSummary = screen.getByRole("complementary", { name: "Session summary" }); expect(preservedSummary).not.toHaveAttribute("aria-hidden", "true"); expect(preservedSummary).toHaveTextContent("Attiva uno"); @@ -133,6 +142,10 @@ test("Pi management preserves the open session summary and the model activity ti await waitFor(() => { expect(screen.getByRole("complementary", { name: "Session summary" })).toHaveTextContent("Domanda originale"); }); + expect(newSession).toHaveAttribute("aria-current", "page"); + expect(newSession).toHaveAttribute("data-navigation-state", "current"); + expect(piManagement).toHaveAttribute("aria-expanded", "false"); + expect(piManagement).toHaveAttribute("data-navigation-state", "available"); await user.click(screen.getByRole("button", { name: "Resume" })); await screen.findByRole("button", { name: "Show model activity" }); @@ -160,17 +173,66 @@ test("administrators can explicitly switch to all sessions and see owners", asyn }), ); wrap(adminUser); - await screen.findByRole("button", { name: "All sessions" }); - expect(screen.getByRole("button", { name: "My sessions" })).toHaveAttribute("aria-pressed", "true"); - expect(screen.getByRole("button", { name: "All sessions" })).toHaveAttribute("aria-pressed", "false"); - await userEvent.click(screen.getByRole("button", { name: "All sessions" })); + const tablist = await screen.findByRole("tablist", { name: "Session scope" }); + const mySessions = within(tablist).getByRole("tab", { name: "My sessions" }); + const allSessions = within(tablist).getByRole("tab", { name: "All sessions" }); + expect(mySessions).toHaveAttribute("aria-selected", "true"); + expect(mySessions).toHaveAttribute("aria-controls", "session-scope-panel"); + expect(mySessions).toHaveAttribute("tabindex", "0"); + expect(mySessions).toHaveAttribute("data-tab-state", "active"); + expect(mySessions).toHaveClass("bg-[oklch(var(--nav-active))]"); + expect(allSessions).toHaveAttribute("aria-selected", "false"); + expect(allSessions).toHaveAttribute("aria-controls", "session-scope-panel"); + expect(allSessions).toHaveAttribute("tabindex", "-1"); + expect(allSessions).toHaveAttribute("data-tab-state", "inactive"); + expect(allSessions).toHaveClass("border-border", "bg-card"); + expect(allSessions).not.toHaveClass("border-transparent", "bg-transparent"); + expect(screen.getByRole("tabpanel")).toHaveAttribute("aria-labelledby", "session-scope-mine-tab"); + await userEvent.click(allSessions); await waitFor(() => expect(scope).toBe("all")); - expect(screen.getByRole("button", { name: "My sessions" })).toHaveAttribute("aria-pressed", "false"); - expect(screen.getByRole("button", { name: "All sessions" })).toHaveAttribute("aria-pressed", "true"); + expect(mySessions).toHaveAttribute("aria-selected", "false"); + expect(mySessions).toHaveAttribute("data-tab-state", "inactive"); + expect(allSessions).toHaveAttribute("aria-selected", "true"); + expect(allSessions).toHaveAttribute("data-tab-state", "active"); + expect(allSessions).toHaveClass("bg-[oklch(var(--nav-active))]"); + expect(screen.getByRole("tabpanel")).toHaveAttribute("aria-labelledby", "session-scope-all-tab"); expect(await screen.findByText("Administrator view: all sessions")).toBeInTheDocument(); expect(screen.getByText("Owner: Bob")).toBeInTheDocument(); }); +test("session scope tabs support roving keyboard navigation", async () => { + let scope = ""; + server.use(http.get("/api/sessions", ({ request }) => { + scope = new URL(request.url).searchParams.get("scope") ?? ""; + return HttpResponse.json(LIST); + })); + wrap(adminUser); + + const user = userEvent.setup(); + const tablist = await screen.findByRole("tablist", { name: "Session scope" }); + const mySessions = within(tablist).getByRole("tab", { name: "My sessions" }); + const allSessions = within(tablist).getByRole("tab", { name: "All sessions" }); + mySessions.focus(); + + await user.keyboard("{ArrowRight}"); + await waitFor(() => expect(scope).toBe("all")); + expect(allSessions).toHaveFocus(); + expect(allSessions).toHaveAttribute("aria-selected", "true"); + expect(mySessions).toHaveAttribute("tabindex", "-1"); + + await user.keyboard("{ArrowLeft}"); + await waitFor(() => expect(scope).toBe("mine")); + expect(mySessions).toHaveFocus(); + expect(mySessions).toHaveAttribute("aria-selected", "true"); + + await user.keyboard("{End}"); + expect(allSessions).toHaveFocus(); + expect(allSessions).toHaveAttribute("aria-selected", "true"); + await user.keyboard("{Home}"); + expect(mySessions).toHaveFocus(); + expect(mySessions).toHaveAttribute("aria-selected", "true"); +}); + test("administrator confirms before deleting a same-named user's session", async () => { let deletes = 0; server.use( @@ -187,7 +249,7 @@ test("administrator confirms before deleting a same-named user's session", async }), ); wrap(adminUser); - await userEvent.click(await screen.findByRole("button", { name: "All sessions" })); + await userEvent.click(await screen.findByRole("tab", { name: "All sessions" })); await screen.findByText("Owner: Alice"); await userEvent.click(screen.getByRole("checkbox", { name: "Select Attiva uno" })); await userEvent.click(screen.getByRole("button", { name: "Delete 1 selected sessions" })); @@ -213,7 +275,7 @@ test("administrator confirms before archiving a same-named user's session", asyn }), ); wrap(adminUser); - await userEvent.click(await screen.findByRole("button", { name: "All sessions" })); + await userEvent.click(await screen.findByRole("tab", { name: "All sessions" })); await screen.findByText("Owner: Alice"); await userEvent.click(screen.getByRole("button", { name: "Session actions" })); await userEvent.click(await screen.findByText("Archive")); diff --git a/frontend/src/shell/AppShell.tsx b/frontend/src/shell/AppShell.tsx index 3f66e79e..be00b3f4 100644 --- a/frontend/src/shell/AppShell.tsx +++ b/frontend/src/shell/AppShell.tsx @@ -17,8 +17,9 @@ import { StopConfirmDialog } from "./StopConfirmDialog"; import { SteerInput, ComposerFooter } from "./SteerInput"; import { WorkflowBar } from "./WorkflowBar"; import { DatabaseManagementPage } from "./DatabaseManagementPage"; -import { Pencil, ArrowLeft, ArrowRight, Trash2 } from "lucide-react"; -import { Button } from "../components/ui/button"; +import { Pencil, ArrowLeft, ArrowRight, ChevronDown, Trash2 } from "lucide-react"; +import { Accordion } from "@base-ui/react/accordion"; +import { Button, buttonVariants } from "../components/ui/button"; import { Checkbox } from "../components/ui/checkbox"; import { Toaster } from "../components/ui/sonner"; import { toast } from "sonner"; @@ -34,7 +35,7 @@ import type { SessionScope, SessionSummary } from "../api/types"; import { useAuthGeneration, useAuthUser } from "../auth/authState"; import { useQuery, useQueryClient } from "@tanstack/react-query"; import { useCallback, useEffect, useMemo, useRef, useState } from "react"; -import type { CSSProperties } from "react"; +import type { CSSProperties, KeyboardEvent } from "react"; import { captureAuthOperation, isAuthOperationCurrent, StaleAuthOperationError, type AuthOperationGuard } from "../auth/authOperation"; interface AppShellProps { @@ -43,6 +44,14 @@ interface AppShellProps { type ActiveSurface = "core" | "database-management"; type ActiveManagementPanel = "workspace" | "pi" | null; +const SESSION_SCOPES: readonly SessionScope[] = ["mine", "all"]; + +function sessionScopeTabClass(selected: boolean): string { + const base = "relative -mb-px flex h-8 w-full cursor-pointer items-center justify-center rounded-t-md border border-b-2 px-2 text-xs font-semibold tracking-[0.005em] outline-none focus-visible:z-10 focus-visible:ring-3 focus-visible:ring-[oklch(var(--nav-active-border)/0.28)]"; + return selected + ? `${base} border-[oklch(var(--nav-active-border))] bg-[oklch(var(--nav-active))] text-[oklch(var(--nav-active-foreground))] hover:bg-[oklch(var(--nav-active-hover))]` + : `${base} border-border bg-card text-muted-foreground hover:border-muted-foreground/45 hover:bg-muted/55 hover:text-foreground`; +} type DatabaseManagementPresentation = "legacy" | "fleet"; @@ -129,8 +138,13 @@ export function AppShell({ canLogout }: AppShellProps) { const [creatingSession, setCreatingSession] = useState(false); const [awaitingQuestion, setAwaitingQuestion] = useState(false); const [sessionScope, setSessionScope] = useState("mine"); + const sessionScopeTabRefs = useRef>({ + mine: null, + all: null, + }); const principal = authenticatedUser; const permissions = authenticatedUser?.permissions ?? []; + const isAdmin = authenticatedUser?.isAdmin === true; const canReadAllSessions = permissions.includes("session.read_all"); const canManageWorkspace = permissions.includes("workspace.manage"); const canManageWorkspaceSecrets = permissions.includes("workspace.secrets.manage"); @@ -164,6 +178,7 @@ export function AppShell({ canLogout }: AppShellProps) { databaseNavigationRef.current = state; }, []); const [activeManagementPanel, setActiveManagementPanel] = useState(null); + const [adminNavigationValue, setAdminNavigationValue] = useState([]); const [activeOpen, setActiveOpen] = useState(true); const [archiveOpen, setArchiveOpen] = useState(false); const [renameTarget, setRenameTarget] = useState(null); @@ -236,6 +251,22 @@ export function AppShell({ canLogout }: AppShellProps) { setSelectedSessionIds(selected ? new Set(sessions.map((session) => session.id)) : new Set()); } + function handleSessionScopeTabKeyDown( + event: KeyboardEvent, + current: SessionScope, + ) { + const currentIndex = SESSION_SCOPES.indexOf(current); + let target: SessionScope | undefined; + if (event.key === "ArrowRight") target = SESSION_SCOPES[(currentIndex + 1) % SESSION_SCOPES.length]; + else if (event.key === "ArrowLeft") target = SESSION_SCOPES[(currentIndex - 1 + SESSION_SCOPES.length) % SESSION_SCOPES.length]; + else if (event.key === "Home") target = SESSION_SCOPES[0]; + else if (event.key === "End") target = SESSION_SCOPES.at(-1); + if (!target) return; + event.preventDefault(); + setSessionScope(target); + sessionScopeTabRefs.current[target]?.focus(); + } + function canLeaveDatabaseManagement(): boolean { if (activeSurface !== "database-management") return true; if (databaseNavigationRef.current.busy) { @@ -627,6 +658,12 @@ export function AppShell({ canLogout }: AppShellProps) { await logoutUser(); } + const currentNavigation = activeSurface === "database-management" + ? "database" + : activeManagementPanel ?? "core"; + const adminNavigationOpen = adminNavigationValue.includes("administration"); + const managementNavigationCurrent = currentNavigation !== "core"; + return (
- - {canManageDatabase && ( - - )} - {canManagePi && ( - + + + + Administration + + + + +
+ + + + + )}
{canReadAllSessions && ( -
-
- - +
- {showingAllSessions && ( -

- Administrator view: all sessions -

- )}
)} - {/* L1 — rail title */} -
- - Sessions - -
- -
- - {selectedSessions.length > 0 && ( - +
+ {canReadAllSessions && showingAllSessions && ( +

+ Administrator view: all sessions +

)} -
-
+ + {/* L1 — rail title */} +
+ + Sessions + +
+ +
+ + {selectedSessions.length > 0 && ( + + )} +
+
{/* L2 — section toggle */}
)} diff --git a/frontend/src/shell/DatabaseManagementPage.test.tsx b/frontend/src/shell/DatabaseManagementPage.test.tsx index 952761ee..a80013c0 100644 --- a/frontend/src/shell/DatabaseManagementPage.test.tsx +++ b/frontend/src/shell/DatabaseManagementPage.test.tsx @@ -29,6 +29,13 @@ function makeDatabase(overrides: Partial = {}): CatalogDatabase workspaceId: "psd-clinical", workspaceName: "Policlinico San Donato", workspaceAvailable: true, + workspaceRevision: { commit: "a".repeat(40), blob: "b".repeat(40) }, + workspaceEvidence: { sourceType: "filesystem", state: "materialized_current_revision" }, + runtimeBinding: { + transport: "postgres_direct", + configurationState: "ready", + sessionTransportSupported: true, + }, configured: true, engine: "postgres", databaseName: "warehouse", @@ -66,6 +73,9 @@ const orphan = makeDatabase({ workspaceId: "retired", workspaceName: "Retired workspace", workspaceAvailable: false, + workspaceRevision: null, + workspaceEvidence: { sourceType: null, state: "workspace_unavailable" }, + runtimeBinding: null, }); function renderPage({ @@ -192,7 +202,7 @@ const synchronizationScopes = [ { scope: "all", label: "Synchronize all" }, ] as const; -test("renders Fleet Ledger with real metrics, conceptual row tooltips, and contextual model selection", async () => { +test("renders Fleet Ledger with real metrics, conceptual row tooltips, and persistent model selection", async () => { const user = userEvent.setup(); server.use( http.get("/api/catalog/metrics", () => HttpResponse.json({ @@ -223,10 +233,15 @@ test("renders Fleet Ledger with real metrics, conceptual row tooltips, and conte expect(screen.queryByText("Thoth catalog · Fleet ledger")).not.toBeInTheDocument(); expect(screen.queryByText("Inspect physical metadata, curate descriptions and control catalog operations.")).not.toBeInTheDocument(); expect(screen.queryByRole("button", { name: "Back to workspace" })).not.toBeInTheDocument(); + expect(screen.queryByRole("button", { name: /Add database/i })).not.toBeInTheDocument(); + expect(screen.getByRole("columnheader", { name: /Revision \/ Evidence/ })).toBeVisible(); + expect(screen.getByRole("columnheader", { name: /NL→SQL runtime/ })).toBeVisible(); + expect(screen.getByRole("columnheader", { name: /Metadata Catalog/ })).toBeVisible(); const summary = screen.getByRole("region", { name: "Fleet summary" }); expect(await within(summary).findByText("2,275")).toBeVisible(); expect(within(summary).getByText("75%")).toBeVisible(); - expect(screen.queryByRole("combobox", { name: "Metadata description model" })).not.toBeInTheDocument(); + const modelSelector = await screen.findByRole("combobox", { name: "Metadata-generation LLM model" }); + expect(modelSelector).toHaveValue("local-qwen"); const databaseRow = await screen.findByRole("row", { name: /Policlinico San Donato/ }); const tablesAction = within(databaseRow).getByRole("button", { name: "View tables for Policlinico San Donato" }); @@ -247,10 +262,103 @@ test("renders Fleet Ledger with real metrics, conceptual row tooltips, and conte const actionPicker = screen.getByRole("combobox", { name: "Batch action" }); expect(within(actionPicker).getByRole("option", { name: /^Synchronize all(?:,|$)/ })).toBeVisible(); await user.selectOptions(actionPicker, "generate-missing"); - expect(await screen.findByRole("combobox", { name: "Metadata description model" })).toHaveValue("local-qwen"); + expect(screen.getAllByRole("combobox", { name: "Metadata-generation LLM model" })).toHaveLength(1); + expect(modelSelector).toHaveValue("local-qwen"); }); +test("shows repository, NL→SQL runtime, and Metadata Catalog states independently", async () => { + const configured = makeDatabase({ connectionStatus: "reachable", testedVersion: 3 }); + const needsRuntimeConfiguration = makeDatabase({ + ...unconfigured, + workspaceEvidence: { sourceType: "http", state: "configuration_required" }, + runtimeBinding: { + transport: "rest_api", + configurationState: "configuration_required", + sessionTransportSupported: true, + }, + }); + renderPage({ rows: [configured, needsRuntimeConfiguration, orphan], presentation: "fleet" }); + + const configuredRow = await screen.findByRole("row", { name: /Policlinico San Donato/ }); + expect(within(configuredRow).getByText("Active revision aaaaaaa")).toBeVisible(); + expect(within(configuredRow).getByText("Evidence materialized · filesystem")).toBeVisible(); + expect(within(configuredRow).getByText("Ready")).toBeVisible(); + expect(within(configuredRow).getByText("Configured")).toBeVisible(); + expect(within(configuredRow).getByText("Connection reachable")).toBeVisible(); + + const needsConfigurationRow = screen.getByRole("row", { name: /Research laboratory/ }); + expect(within(needsConfigurationRow).getByText("Evidence credentials required · http")).toBeVisible(); + expect(within(needsConfigurationRow).getByText("Configuration required")).toBeVisible(); + expect(within(needsConfigurationRow).getByText("Not configured")).toBeVisible(); + + const orphanRow = screen.getByRole("row", { name: /Retired workspace/ }); + expect(within(orphanRow).getByText("Workspace missing")).toBeVisible(); + expect(within(orphanRow).getByText("Unavailable")).toBeVisible(); + expect(within(orphanRow).getByText("Orphaned configuration")).toBeVisible(); +}); + +test("keeps Fleet Ledger visible when a catalog response omits the Evidence projection", async () => { + const legacyDatabase = makeDatabase(); + delete (legacyDatabase as Partial).workspaceEvidence; + + renderPage({ rows: [legacyDatabase], presentation: "fleet" }); + + expect(await screen.findByRole("heading", { name: "Database management" })).toBeVisible(); + const databaseRow = await screen.findByRole("row", { name: /Policlinico San Donato/ }); + expect(within(databaseRow).getByText("Evidence state unavailable")).toBeVisible(); +}); + +test("opens the relationship map directly from a Fleet database and restores focus on return", async () => { + const user = userEvent.setup(); + server.use( + http.get("/api/catalog/databases/:databaseId/relationships", () => HttpResponse.json([])), + ); + renderPage({ + rows: [makeDatabase({ connectionStatus: "reachable", testedVersion: 3 })], + presentation: "fleet", + }); + + const databaseRow = await screen.findByRole("row", { name: /Policlinico San Donato/ }); + const relationshipsAction = within(databaseRow).getByRole("button", { + name: "View relationships for Policlinico San Donato", + }); + expect(relationshipsAction.parentElement).toHaveAttribute("data-tooltip", "Relationships"); + expect(relationshipsAction).not.toHaveAttribute("title"); + + await user.click(relationshipsAction); + expect(await screen.findByRole("region", { + name: "Relationships for Policlinico San Donato", + })).toBeVisible(); + expect(screen.getByText("Relationship map")).toBeVisible(); + + await user.click(screen.getByRole("button", { name: "Back to databases" })); + await waitFor(() => expect(screen.getByRole("button", { + name: "View relationships for Policlinico San Donato", + })).toHaveFocus()); +}); + +test("offers catalog configuration directly on an unconfigured Fleet workspace", async () => { + const user = userEvent.setup(); + renderPage({ rows: [unconfigured], presentation: "fleet" }); + + const databaseRow = await screen.findByRole("row", { name: /Research laboratory/ }); + expect(within(databaseRow).queryByRole("button", { + name: "View relationships for Research laboratory", + })).not.toBeInTheDocument(); + expect(within(databaseRow).queryByRole("button", { + name: "View tables for Research laboratory", + })).not.toBeInTheDocument(); + + await user.click(within(databaseRow).getByRole("button", { + name: "Configure catalog for Research laboratory", + })); + + const drawer = await screen.findByRole("dialog", { name: "Configure catalog" }); + expect(within(drawer).getByLabelText("Workspace")).toHaveAttribute("readonly"); + expect(within(drawer).getByLabelText("Workspace")).toHaveValue("Research laboratory"); +}); + test("keeps only the local back action while browsing Fleet Ledger columns", async () => { const user = userEvent.setup(); server.use( @@ -276,6 +384,32 @@ test("keeps only the local back action while browsing Fleet Ledger columns", asy expect(screen.queryByRole("button", { name: "Back to tables" })).not.toBeInTheDocument(); }); +test("uses the database back-action background treatment for the table back action", async () => { + const user = userEvent.setup(); + server.use( + http.get("/api/catalog/databases/:databaseId/tables", () => HttpResponse.json([patientsTable])), + http.get("/api/catalog/databases/:databaseId/tables/:tableId/columns", () => HttpResponse.json([])), + ); + renderPage({ rows: [makeDatabase()], presentation: "fleet" }); + + const databaseRow = await screen.findByRole("row", { name: /Policlinico San Donato/ }); + await user.click(within(databaseRow).getByRole("button", { + name: "View tables for Policlinico San Donato", + })); + const databaseBackAction = await screen.findByRole("button", { name: "Back to databases" }); + expect(databaseBackAction).toHaveClass("thot-fleet-back-action"); + const backgroundTreatment = (element: HTMLElement) => element.className + .split(/\s+/) + .filter((className) => /(^|:)bg-/.test(className)) + .sort(); + + await user.click(await screen.findByRole("button", { name: "View columns for patients" })); + + const tableBackAction = await screen.findByRole("button", { name: "Back to tables" }); + expect(tableBackAction).toHaveClass("thot-fleet-back-action"); + expect(backgroundTreatment(tableBackAction)).toEqual(backgroundTreatment(databaseBackAction)); +}); + test("reopens database synchronization history when no run is active", async () => { const user = userEvent.setup(); const completed: CatalogSyncRun = { @@ -312,7 +446,7 @@ test("selects the configured metadata-description model by default", async () => }))); renderPage(); - const selector = await screen.findByRole("combobox", { name: "Metadata description model" }); + const selector = await screen.findByRole("combobox", { name: "Metadata-generation LLM model" }); await waitFor(() => expect(selector).toHaveValue("local-qwen")); }); @@ -320,7 +454,7 @@ test("keeps both run-history buttons visible beside the metadata-description sel const user = userEvent.setup(); renderPage(); - const selector = await screen.findByRole("combobox", { name: "Metadata description model" }); + const selector = await screen.findByRole("combobox", { name: "Metadata-generation LLM model" }); const toolbar = screen.getByRole("group", { name: "Database management controls" }); const metadataControls = screen.getByRole("group", { name: "Metadata description controls" }); const actions = screen.getByRole("group", { name: "Database management actions" }); @@ -344,7 +478,7 @@ test("keeps both run-history buttons visible beside the metadata-description sel expect(toolbar.lastElementChild).toBe(actions); expect(actions).toHaveClass("sm:justify-end"); expect(actions).toContainElement(screen.getByRole("button", { name: "Refresh" })); - expect(actions).toContainElement(screen.getByRole("button", { name: "Add database" })); + expect(within(actions).queryByRole("button", { name: /Add database/i })).not.toBeInTheDocument(); expect(screen.queryByRole("note", { name: "Metadata generation source data disclosure", })).not.toBeInTheDocument(); @@ -371,7 +505,7 @@ test("changes the metadata-description model in page-local state", async () => { }))); renderPage(); - const selector = await screen.findByRole("combobox", { name: "Metadata description model" }); + const selector = await screen.findByRole("combobox", { name: "Metadata-generation LLM model" }); await waitFor(() => expect(selector).toHaveValue("local-qwen")); await user.selectOptions(selector, "openai-mini"); @@ -385,10 +519,10 @@ test("explains application setup when no metadata-description model is configure }))); renderPage(); - const selector = await screen.findByRole("combobox", { name: "Metadata description model" }); + const selector = await screen.findByRole("combobox", { name: "Metadata-generation LLM model" }); expect(selector).toBeDisabled(); expect(await screen.findByText( - "Configure a metadata-generation model in application setup to enable description generation.", + "No metadata-generation LLM model is configured for this installation.", )).toBeVisible(); }); @@ -399,11 +533,11 @@ test("uses a safe disabled setup state when metadata-description models are unav }, { status: 503 }))); renderPage(); - const selector = await screen.findByRole("combobox", { name: "Metadata description model" }); + const selector = await screen.findByRole("combobox", { name: "Metadata-generation LLM model" }); await waitFor(() => expect(selector).toHaveDisplayValue("Model choices unavailable")); expect(selector).toBeDisabled(); expect(screen.getByText( - "Configure a metadata-generation model in application setup to enable description generation.", + "Metadata-generation LLM model choices are unavailable.", )).toBeVisible(); expect(screen.queryByText("internal model registry details")).not.toBeInTheDocument(); }); @@ -417,11 +551,11 @@ test.each([null, "missing-model"])( }))); renderPage(); - const selector = await screen.findByRole("combobox", { name: "Metadata description model" }); + const selector = await screen.findByRole("combobox", { name: "Metadata-generation LLM model" }); await waitFor(() => expect(selector).toHaveDisplayValue("No usable model configured")); expect(selector).toBeDisabled(); expect(screen.getByText( - "Configure a metadata-generation model in application setup to enable description generation.", + "No metadata-generation LLM model is configured for this installation.", )).toBeVisible(); }, ); @@ -441,7 +575,7 @@ test("renders only the public metadata-generation model contract", async () => { }))); renderPage(); - const selector = await screen.findByRole("combobox", { name: "Metadata description model" }); + const selector = await screen.findByRole("combobox", { name: "Metadata-generation LLM model" }); await waitFor(() => expect(selector).toHaveValue("approved-model")); expect(within(selector).getAllByRole("option").map((option) => option.textContent)) .toEqual(["Approved model"]); @@ -458,8 +592,9 @@ test("starts with a full-width list and applies the row action matrix", async () expect(screen.getByRole("button", { name: "Delete Policlinico San Donato" })).toBeEnabled(); expect(screen.getByRole("button", { name: "View Research laboratory" })).toBeEnabled(); - expect(screen.getByRole("button", { name: "Edit Research laboratory" })).toBeEnabled(); - expect(screen.getByRole("button", { name: "Delete Research laboratory" })).toBeDisabled(); + expect(screen.getByRole("button", { name: "Configure catalog for Research laboratory" })).toBeEnabled(); + expect(screen.queryByRole("button", { name: "Edit Research laboratory" })).not.toBeInTheDocument(); + expect(screen.queryByRole("button", { name: "Delete Research laboratory" })).not.toBeInTheDocument(); expect(screen.getByRole("button", { name: "View Retired workspace" })).toBeEnabled(); expect(screen.getByRole("button", { name: "Edit Retired workspace" })).toBeDisabled(); @@ -558,7 +693,7 @@ test("keeps Fleet Generate Missing behind the source-data disclosure", async () "generate-missing", ); await waitFor(() => expect(screen.getByRole("combobox", { - name: "Metadata description model", + name: "Metadata-generation LLM model", })).toHaveValue("local-qwen")); await user.click(screen.getByRole("button", { name: "Run action" })); @@ -951,7 +1086,7 @@ test("replaces the list with View and returns focus to the originating action", expect(screen.getByRole("button", { name: "View Policlinico San Donato" })).toBeVisible(); }); -test("opens an unconfigured row as Edit while creating its first saved configuration", async () => { +test("configures the catalog for the selected workspace and locks that workspace", async () => { const user = userEvent.setup(); let postBody: unknown; const saved = makeDatabase({ @@ -971,16 +1106,16 @@ test("opens an unconfigured row as Edit while creating its first saved configura ); renderPage({ rows: () => rows }); - await user.click(await screen.findByRole("button", { name: "Edit Research laboratory" })); + await user.click(await screen.findByRole("button", { name: "Configure catalog for Research laboratory" })); - expect(screen.getByRole("heading", { name: "Edit database" })).toBeVisible(); - expect(screen.getByLabelText("Workspace")).toBeDisabled(); + expect(screen.getByRole("heading", { name: "Configure catalog" })).toBeVisible(); + expect(screen.getByLabelText("Workspace")).toHaveAttribute("readonly"); await user.selectOptions(screen.getByLabelText("Transport"), "rest_api"); await user.type(screen.getByLabelText("Base URL"), "https://psd.example/api"); expect(screen.getByLabelText("Diagnostic endpoint")).toHaveValue("/health"); expect(screen.getByLabelText("Diagnostic endpoint")).toHaveAttribute("readonly"); - await user.click(screen.getByRole("button", { name: "Save database" })); + await user.click(screen.getByRole("button", { name: "Save catalog configuration" })); await waitFor(() => expect(postBody).toMatchObject({ workspaceId: "lab", @@ -989,7 +1124,7 @@ test("opens an unconfigured row as Edit while creating its first saved configura expect(await screen.findByRole("heading", { name: "Edit database" })).toBeVisible(); }); -test("global Add offers only unconfigured workspaces and lets the operator choose one", async () => { +test("has no global database creation path and scopes configuration to the chosen row", async () => { const user = userEvent.setup(); const second = makeDatabase({ ...unconfigured, @@ -999,14 +1134,13 @@ test("global Add offers only unconfigured workspaces and lets the operator choos }); renderPage({ rows: [makeDatabase(), unconfigured, second] }); - await user.click(await screen.findByRole("button", { name: "Add database" })); + expect(screen.queryByRole("button", { name: /Add database/i })).not.toBeInTheDocument(); + await user.click(await screen.findByRole("button", { name: "Configure catalog for Radiology" })); - expect(screen.getByRole("heading", { name: "Add database" })).toBeVisible(); - expect(screen.getByRole("button", { name: "Add database" })).toBeVisible(); - const selector = screen.getByLabelText("Workspace"); - expect(selector).toBeEnabled(); - expect(screen.queryByRole("option", { name: "Policlinico San Donato" })).not.toBeInTheDocument(); - await user.selectOptions(selector, "radiology"); + expect(screen.getByRole("heading", { name: "Configure catalog" })).toBeVisible(); + const workspace = screen.getByLabelText("Workspace"); + expect(workspace).toHaveAttribute("readonly"); + expect(workspace).toHaveValue("Radiology"); expect(screen.getByDisplayValue("radiology_dwh")).toBeVisible(); }); @@ -1064,7 +1198,8 @@ test("uses an in-page destructive form and returns the YAML workspace to Not con await waitFor(() => expect(deleteCalled).toBe(true)); expect(deleteVersion).toBe("3"); - expect(await screen.findByRole("button", { name: "Delete Policlinico San Donato" })).toBeDisabled(); + expect(await screen.findByRole("button", { name: "Configure catalog for Policlinico San Donato" })).toBeEnabled(); + expect(screen.queryByRole("button", { name: "Delete Policlinico San Donato" })).not.toBeInTheDocument(); expect(screen.getByText("Not configured")).toBeVisible(); }); @@ -2677,6 +2812,8 @@ test("navigates from a table to columns and from the database to physical relati deferrable: false, initiallyDeferred: false, columns: [{ position: 1, sourceColumnId: "1", sourceColumnName: "patient_id", targetColumnId: idColumn.id, targetColumnName: "id" }], + origin: "physical", + status: "active", lastSyncedDatabaseVersion: 3, lastSyncedAt: "2026-08-27T10:00:00Z", createdAt: "2026-08-27T10:00:00Z", diff --git a/frontend/src/shell/DatabaseManagementPage.tsx b/frontend/src/shell/DatabaseManagementPage.tsx index 1235377e..12f7d9b2 100644 --- a/frontend/src/shell/DatabaseManagementPage.tsx +++ b/frontend/src/shell/DatabaseManagementPage.tsx @@ -6,7 +6,7 @@ import { useState, } from "react"; import { useQuery, useQueryClient } from "@tanstack/react-query"; -import { History, Plus, RefreshCw } from "lucide-react"; +import { History, RefreshCw } from "lucide-react"; import { toast } from "sonner"; import { Button } from "../components/ui/button"; import { ApiError, apiErrorMessage } from "../api/client"; @@ -209,11 +209,7 @@ export function DatabaseManagementPage({ () => queryClient.invalidateQueries({ queryKey: ["catalog-metrics"] }), [queryClient], ); - const availableWorkspaces = useMemo( - () => rows.filter((row) => row.workspaceAvailable && !row.configured), - [rows], - ); - const editable = screen.kind === "add" || screen.kind === "edit"; + const editable = screen.kind === "configure" || screen.kind === "edit"; const configurationDirty = Boolean( editable && draft @@ -221,8 +217,9 @@ export function DatabaseManagementPage({ ); const dirty = Boolean(editable && draft && (configurationDirty || hasSecretChanges(draft))); const busy = busyAction !== null; - const navigationDirty = screen.kind === "tables" ? tablesNavigationState.dirty : dirty; - const navigationBusy = screen.kind === "tables" ? tablesNavigationState.busy : busy; + const catalogChildVisible = screen.kind === "tables" || screen.kind === "relationships"; + const navigationDirty = catalogChildVisible ? tablesNavigationState.dirty : dirty; + const navigationBusy = catalogChildVisible ? tablesNavigationState.busy : busy; useEffect(() => { onNavigationStateChange?.({ dirty: navigationDirty, busy: navigationBusy }); @@ -263,9 +260,14 @@ export function DatabaseManagementPage({ }, [queryClient]); const restoreListFocus = useCallback(() => { + const originLabel = originRef.current?.getAttribute("aria-label"); window.setTimeout(() => { if (originRef.current?.isConnected) { originRef.current.focus(); + } else if (originLabel) { + const matchingAction = [...document.querySelectorAll("button")] + .find((button) => button.getAttribute("aria-label") === originLabel); + (matchingAction ?? searchInputRef.current)?.focus(); } else { searchInputRef.current?.focus(); } @@ -307,7 +309,6 @@ export function DatabaseManagementPage({ mode: DatabaseFormMode, row: CatalogDatabase, origin: HTMLElement, - workspaceLocked = false, ) => { const nextDraft = draftFrom(row); originRef.current = origin; @@ -320,7 +321,6 @@ export function DatabaseManagementPage({ setScreen({ kind: mode, workspaceId: row.workspaceId, - ...(mode === "add" ? { workspaceLocked } : {}), }); }, []); @@ -329,7 +329,7 @@ export function DatabaseManagementPage({ }, [openForm]); const editRow = useCallback((row: CatalogDatabase, origin: HTMLButtonElement) => { - openForm(row.configured ? "edit" : "add", row, origin, !row.configured); + openForm(row.configured ? "edit" : "configure", row, origin); }, [openForm]); const deleteRow = useCallback((row: CatalogDatabase, origin: HTMLButtonElement) => { @@ -349,8 +349,9 @@ export function DatabaseManagementPage({ setScreen({ kind: "tables", workspaceId: row.workspaceId }); }, []); - const openRelationships = useCallback((row: CatalogDatabase) => { + const openRelationships = useCallback((row: CatalogDatabase, origin?: HTMLElement) => { if (!row.configured || !row.id) return; + if (origin) originRef.current = origin; setDraft(null); setBaseline(""); setFormSource(null); @@ -372,25 +373,6 @@ export function DatabaseManagementPage({ setScreen({ kind: "view", workspaceId: row.workspaceId }); }, []); - const addDatabase = useCallback((origin: HTMLElement) => { - const workspace = availableWorkspaces[0]; - if (!workspace || !canManage) return; - openForm("add", workspace, origin, false); - }, [availableWorkspaces, canManage, openForm]); - - const changeWorkspace = useCallback((workspaceId: string) => { - const workspace = availableWorkspaces.find((row) => row.workspaceId === workspaceId); - if (!workspace) return; - if (dirty && !window.confirm("Discard changes and choose another workspace?")) return; - const nextDraft = draftFrom(workspace); - setDraft(nextDraft); - setBaseline(configurationFingerprint(nextDraft)); - setFormSource({ id: workspace.id, version: workspace.version }); - setStale(false); - setPartialSecretFailure(null); - setScreen({ kind: "add", workspaceId, workspaceLocked: false }); - }, [availableWorkspaces, dirty]); - const changeField = useCallback((field: "databaseName" | "schema", value: string) => { setDraft((current) => current ? { ...current, [field]: value } : current); }, []); @@ -624,7 +606,7 @@ export function DatabaseManagementPage({ showList(); toast.info("This database configuration has already been deleted"); } else if (screen.kind === "edit" && !latest.configured) { - setScreen({ kind: "add", workspaceId: latest.workspaceId, workspaceLocked: true }); + setScreen({ kind: "configure", workspaceId: latest.workspaceId }); } } catch (error) { toast.error(apiErrorMessage(error)); @@ -669,22 +651,40 @@ export function DatabaseManagementPage({ }, [queryClient]); const openDescriptionGenerationHistory = useCallback(() => { - const run = observedActiveDescriptionGenerationRun - ?? descriptionGenerationRuns[0] - ?? activeDescriptionGenerationRun; + const scopedRuns = activeRow ? descriptionGenerationRuns.filter((item) => item.databaseId === activeRow.id) : descriptionGenerationRuns; + const scopedActiveRun = observedActiveDescriptionGenerationRun + && (!activeRow || observedActiveDescriptionGenerationRun.databaseId === activeRow.id) + ? observedActiveDescriptionGenerationRun + : undefined; + const scopedSelectedRun = activeDescriptionGenerationRun + && (!activeRow || activeDescriptionGenerationRun.databaseId === activeRow.id) + ? activeDescriptionGenerationRun + : undefined; + const run = scopedActiveRun + ?? scopedRuns[0] + ?? scopedSelectedRun; if (run) setActiveDescriptionGenerationRun(run); setSensitiveDataSuggestionHistoryDrawerOpen(false); setDescriptionGenerationDrawerOpen(true); - }, [activeDescriptionGenerationRun, descriptionGenerationRuns, observedActiveDescriptionGenerationRun]); + }, [activeDescriptionGenerationRun, activeRow, descriptionGenerationRuns, observedActiveDescriptionGenerationRun]); const openSensitiveDataSuggestionHistory = useCallback(() => { - const run = observedActiveSensitiveDataSuggestionRun - ?? sensitiveDataSuggestionRuns[0] - ?? activeSensitiveDataSuggestionRun; + const scopedRuns = activeRow ? sensitiveDataSuggestionRuns.filter((item) => item.databaseId === activeRow.id) : sensitiveDataSuggestionRuns; + const scopedActiveRun = observedActiveSensitiveDataSuggestionRun + && (!activeRow || observedActiveSensitiveDataSuggestionRun.databaseId === activeRow.id) + ? observedActiveSensitiveDataSuggestionRun + : undefined; + const scopedSelectedRun = activeSensitiveDataSuggestionRun + && (!activeRow || activeSensitiveDataSuggestionRun.databaseId === activeRow.id) + ? activeSensitiveDataSuggestionRun + : undefined; + const run = scopedActiveRun + ?? scopedRuns[0] + ?? scopedSelectedRun; if (run) setActiveSensitiveDataSuggestionRun(run); setDescriptionGenerationDrawerOpen(false); setSensitiveDataSuggestionHistoryDrawerOpen(true); - }, [activeSensitiveDataSuggestionRun, observedActiveSensitiveDataSuggestionRun, sensitiveDataSuggestionRuns]); + }, [activeRow, activeSensitiveDataSuggestionRun, observedActiveSensitiveDataSuggestionRun, sensitiveDataSuggestionRuns]); const descriptionGenerationTerminated = useCallback(async (run: DescriptionGenerationRun) => { await Promise.all([ @@ -902,6 +902,7 @@ export function DatabaseManagementPage({ isLoading={metadataModelsQuery.isLoading} selectedModel={selectedMetadataModel} onSelectedModelChange={setSelectedMetadataModel} + variant="compact" /> ); @@ -943,6 +944,7 @@ export function DatabaseManagementPage({ /> model.id === activeDescriptionGenerationRun?.modelId, @@ -953,6 +955,7 @@ export function DatabaseManagementPage({ /> model.id === activeSensitiveDataSuggestionRun?.modelId, @@ -980,7 +983,6 @@ export function DatabaseManagementPage({ eyebrow="Catalog configuration" title={databaseFormTitle( fleetFormMode, - fleetFormMode === "add" && Boolean(screen.kind === "add" && screen.workspaceLocked), "drawer", )} description={`${activeRow.workspaceName} · ${activeRow.workspaceId}`} @@ -993,8 +995,6 @@ export function DatabaseManagementPage({ presentation="drawer" mode={fleetFormMode} row={activeRow} - availableWorkspaces={availableWorkspaces} - workspaceLocked={fleetFormMode === "add" && Boolean(screen.kind === "add" && screen.workspaceLocked)} draft={draft} dirty={dirty} canManage={canManage} @@ -1005,7 +1005,6 @@ export function DatabaseManagementPage({ partialSecretFailure={partialSecretFailure} headingRef={formHeadingRef} onBack={backToList} - onWorkspaceChange={changeWorkspace} onFieldChange={changeField} onTransportChange={changeTransport} onBindingChange={changeBinding} @@ -1020,6 +1019,8 @@ export function DatabaseManagementPage({ onOpenRelationships={() => openRelationships(activeRow)} onSync={(scope) => void syncDatabase(scope)} onOpenSync={() => openSync(activeRow)} + onOpenDescriptionHistory={openDescriptionGenerationHistory} + onOpenSensitiveHistory={openSensitiveDataSuggestionHistory} activeSyncRun={currentActiveRun} /> @@ -1040,10 +1041,11 @@ export function DatabaseManagementPage({ onNestedNavigationChange={setTableNestedNavigationActive} onRunStarted={rememberSyncRun} onOpenSync={() => openSync(activeRow)} + onOpenDescriptionHistory={openDescriptionGenerationHistory} + onOpenSensitiveHistory={openSensitiveDataSuggestionHistory} onDescriptionGenerationRunStarted={rememberDescriptionGenerationRun} onSuggestSensitive={suggestActiveDatabaseSensitiveFields} onCatalogMetricsChanged={invalidateCatalogMetrics} - metadataModelControl={fleetMetadataModelControl} /> ) : screen.kind === "relationships" && relationshipsVisible ? ( openTables(activeRow)} onRunStarted={rememberSyncRun} onOpenSync={() => openSync(activeRow)} + onOpenDescriptionHistory={openDescriptionGenerationHistory} + onOpenSensitiveHistory={openSensitiveDataSuggestionHistory} + onCatalogMetricsChanged={invalidateCatalogMetrics} + onNavigationStateChange={setTablesNavigationState} /> ) : isError && rows.length === 0 ? (
@@ -1078,6 +1084,7 @@ export function DatabaseManagementPage({ onSearchChange={setSearch} onView={viewRow} onOpenTables={(row, origin) => openTables(row, origin)} + onOpenRelationships={(row, origin) => openRelationships(row, origin)} onEdit={editRow} onDelete={deleteRow} onOpenSync={openSync} @@ -1088,7 +1095,6 @@ export function DatabaseManagementPage({ onGenerateDescriptions={generateDatabaseDescriptions} onSuggestSensitive={suggestDatabaseSensitiveFields} onDeleteMetadataSelected={deleteSelectedMetadata} - metadataModelControl={fleetMetadataModelControl} onRefresh={refreshList} /> ); @@ -1118,28 +1124,7 @@ export function DatabaseManagementPage({ )} actions={( <> - - - {(screen.kind === "list" || formVisible) ? ( - - ) : null} + {fleetMetadataModelControl} )} /> @@ -1205,40 +1190,16 @@ export function DatabaseManagementPage({ selectedModel={selectedMetadataModel} onSelectedModelChange={setSelectedMetadataModel} /> - - + {screen.kind === "list" ? <> + + + : null}
{screen.kind === "list" ? (
void refreshList()}> Refresh -
) : null}
@@ -1308,8 +1261,6 @@ export function DatabaseManagementPage({ openRelationships(activeRow)} onSync={(scope) => void syncDatabase(scope)} onOpenSync={() => openSync(activeRow)} + onOpenDescriptionHistory={openDescriptionGenerationHistory} + onOpenSensitiveHistory={openSensitiveDataSuggestionHistory} activeSyncRun={currentActiveRun} /> ) : null} @@ -1352,6 +1304,8 @@ export function DatabaseManagementPage({ onNavigationStateChange={setTablesNavigationState} onRunStarted={rememberSyncRun} onOpenSync={() => openSync(activeRow)} + onOpenDescriptionHistory={openDescriptionGenerationHistory} + onOpenSensitiveHistory={openSensitiveDataSuggestionHistory} onDescriptionGenerationRunStarted={rememberDescriptionGenerationRun} onSuggestSensitive={suggestActiveDatabaseSensitiveFields} onCatalogMetricsChanged={invalidateCatalogMetrics} @@ -1367,6 +1321,10 @@ export function DatabaseManagementPage({ onOpenTables={() => openTables(activeRow)} onRunStarted={rememberSyncRun} onOpenSync={() => openSync(activeRow)} + onOpenDescriptionHistory={openDescriptionGenerationHistory} + onOpenSensitiveHistory={openSensitiveDataSuggestionHistory} + onCatalogMetricsChanged={invalidateCatalogMetrics} + onNavigationStateChange={setTablesNavigationState} /> ) : null}
@@ -1381,6 +1339,7 @@ export function DatabaseManagementPage({ /> model.id === activeDescriptionGenerationRun?.modelId, @@ -1391,6 +1350,7 @@ export function DatabaseManagementPage({ /> model.id === activeSensitiveDataSuggestionRun?.modelId, diff --git a/frontend/src/shell/database-management/CatalogSyncDrawer.tsx b/frontend/src/shell/database-management/CatalogSyncDrawer.tsx index 71e2ef0b..41171808 100644 --- a/frontend/src/shell/database-management/CatalogSyncDrawer.tsx +++ b/frontend/src/shell/database-management/CatalogSyncDrawer.tsx @@ -249,6 +249,7 @@ export function CatalogSyncDrawer({ key={item.id} type="button" aria-current={item.id === run?.id ? "true" : undefined} + data-status={item.state} className="flex w-full items-center justify-between gap-3 px-3 py-2 text-left text-sm hover:bg-muted/50 aria-[current=true]:bg-primary/8" onClick={() => onRunChange(item)} > diff --git a/frontend/src/shell/database-management/DatabaseColumns.tsx b/frontend/src/shell/database-management/DatabaseColumns.tsx index 2e0b13b3..c2897469 100644 --- a/frontend/src/shell/database-management/DatabaseColumns.tsx +++ b/frontend/src/shell/database-management/DatabaseColumns.tsx @@ -1,4 +1,4 @@ -import { useEffect, useMemo, useRef, useState, type ReactNode } from "react"; +import { useEffect, useMemo, useRef, useState } from "react"; import { Menu } from "@base-ui/react/menu"; import { useQuery, useQueryClient } from "@tanstack/react-query"; import { AgGridReact } from "ag-grid-react"; @@ -20,6 +20,7 @@ import { import type { DatabaseNavigationState } from "./model"; import { FleetActionSelector, type FleetActionOption } from "./FleetActionSelector"; import { FleetLedgerDrawer } from "./FleetLedgerShell"; +import { NO_METADATA_GENERATION_LLM_MODEL_MESSAGE } from "./MetadataGenerationModelSelector"; interface Props { databaseId: string; @@ -34,7 +35,6 @@ interface Props { bindingReady?: boolean; catalogOperationActive?: boolean; onCatalogMetricsChanged?: () => void | Promise; - metadataModelControl?: ReactNode; presentation?: "legacy" | "fleet"; } @@ -99,7 +99,6 @@ export function DatabaseColumns({ bindingReady = true, catalogOperationActive = false, onCatalogMetricsChanged, - metadataModelControl, presentation = "legacy", }: Props) { const queryClient = useQueryClient(); @@ -307,7 +306,7 @@ export function DatabaseColumns({ : selectedIds.length === 0 ? "Select at least one column." : !selectedMetadataModel - ? "Choose a metadata model first." + ? NO_METADATA_GENERATION_LLM_MODEL_MESSAGE : descriptionGenerationActive || catalogOperationActive ? "Wait for the active catalog operation to finish." : busy @@ -341,7 +340,7 @@ export function DatabaseColumns({ : selectedIds.length === 0 ? "Select at least one column." : !selectedMetadataModel - ? "Choose a metadata model first." + ? NO_METADATA_GENERATION_LLM_MODEL_MESSAGE : descriptionGenerationActive || catalogOperationActive ? "Wait for the active catalog operation to finish." : busy @@ -468,7 +467,6 @@ export function DatabaseColumns({ busyLabel={sensitiveAction === "suggest" ? "Suggesting…" : sensitiveAction === "save" ? "Saving…" : "Running…"} onRun={runFleetAction} onClear={clearSelection} - renderContext={(action) => action === "generate-descriptions" || action === "suggest-sensitive" ? metadataModelControl : null} label="Column action" /> @@ -501,7 +499,6 @@ export function DatabaseColumns({ busyLabel={sensitiveAction === "save" ? "Saving…" : "Running…"} onRun={runFleetAction} onClear={clearSelection} - renderContext={(action) => action === "generate-descriptions" || action === "suggest-sensitive" ? metadataModelControl : null} label="Column action" /> setSearch(event.target.value)} /> diff --git a/frontend/src/shell/database-management/DatabaseFleetQueryErrors.test.tsx b/frontend/src/shell/database-management/DatabaseFleetQueryErrors.test.tsx index 17ba030f..b6c47306 100644 --- a/frontend/src/shell/database-management/DatabaseFleetQueryErrors.test.tsx +++ b/frontend/src/shell/database-management/DatabaseFleetQueryErrors.test.tsx @@ -14,6 +14,9 @@ const database: CatalogDatabase = { workspaceId: "psd-clinical", workspaceName: "Policlinico San Donato", workspaceAvailable: true, + workspaceRevision: { commit: "a".repeat(40), blob: "b".repeat(40) }, + workspaceEvidence: { sourceType: "filesystem", state: "materialized_current_revision" }, + runtimeBinding: { transport: "postgres_direct", configurationState: "ready", sessionTransportSupported: true }, configured: true, engine: "postgres", databaseName: "warehouse", diff --git a/frontend/src/shell/database-management/DatabaseForm.tsx b/frontend/src/shell/database-management/DatabaseForm.tsx index a34067fa..af198f26 100644 --- a/frontend/src/shell/database-management/DatabaseForm.tsx +++ b/frontend/src/shell/database-management/DatabaseForm.tsx @@ -113,8 +113,6 @@ function SecretField({ interface DatabaseFormProps { mode: DatabaseFormMode; row: CatalogDatabase; - availableWorkspaces: CatalogDatabase[]; - workspaceLocked: boolean; draft: DatabaseFormDraft; dirty: boolean; canManage: boolean; @@ -125,7 +123,6 @@ interface DatabaseFormProps { partialSecretFailure: string | null; headingRef: RefObject; onBack: () => void; - onWorkspaceChange: (workspaceId: string) => void; onFieldChange: (field: "databaseName" | "schema", value: string) => void; onTransportChange: (transport: DatabaseTransport) => void; onBindingChange: (key: K, value: DatabaseBinding[K]) => void; @@ -140,16 +137,17 @@ interface DatabaseFormProps { onOpenRelationships: () => void; onSync: (scope: CatalogSyncScope) => void; onOpenSync: () => void; + onOpenDescriptionHistory?: () => void; + onOpenSensitiveHistory?: () => void; activeSyncRun?: CatalogSyncRun; presentation?: "page" | "drawer"; } export function databaseFormTitle( mode: DatabaseFormMode, - workspaceLocked: boolean, presentation: "page" | "drawer" = "page", ): string { - if (mode === "add") return workspaceLocked ? "Edit database" : "Add database"; + if (mode === "configure") return "Configure catalog"; if (mode === "edit") return "Edit database"; if (mode === "delete") return presentation === "drawer" ? "Remove configuration" : "Delete database"; return "Database details"; @@ -158,8 +156,6 @@ export function databaseFormTitle( export function DatabaseForm({ mode, row, - availableWorkspaces, - workspaceLocked, draft, dirty, canManage, @@ -170,7 +166,6 @@ export function DatabaseForm({ partialSecretFailure, headingRef, onBack, - onWorkspaceChange, onFieldChange, onTransportChange, onBindingChange, @@ -185,12 +180,14 @@ export function DatabaseForm({ onOpenRelationships, onSync, onOpenSync, + onOpenDescriptionHistory, + onOpenSensitiveHistory, activeSyncRun, presentation = "page", }: DatabaseFormProps) { - const title = databaseFormTitle(mode, workspaceLocked, presentation); + const title = databaseFormTitle(mode, presentation); const readOnly = mode === "view" || mode === "delete"; - const editable = mode === "add" || mode === "edit"; + const editable = mode === "configure" || mode === "edit"; const busy = busyAction !== null; const testDisabled = !canManage || !row.configured @@ -268,27 +265,9 @@ export function DatabaseForm({ ) : null}
- {mode === "add" ? ( - - - - ) : ( - - - - )} + + + @@ -420,13 +399,13 @@ export function DatabaseForm({ {busyAction === "test" ? "Testing…" : "Test connection"} ) : null} - {mode === "add" || mode === "edit" ? ( + {mode === "configure" || mode === "edit" ? ( ) : null} @@ -459,6 +438,16 @@ export function DatabaseForm({ {activeSyncRun ? : } Sync history ) : null} + {mode === "view" && row.configured && onOpenDescriptionHistory ? ( + + ) : null} + {mode === "view" && row.configured && onOpenSensitiveHistory ? ( + + ) : null} {mode === "view" ? ( ) : null} + {mode === "view" && row.configured && onOpenDescriptionHistory ? : null} + {mode === "view" && row.configured && onOpenSensitiveHistory ? : null}
{row.lastTestedAt ? ( diff --git a/frontend/src/shell/database-management/DatabaseGrid.tsx b/frontend/src/shell/database-management/DatabaseGrid.tsx index 128c9f96..cd33f2f7 100644 --- a/frontend/src/shell/database-management/DatabaseGrid.tsx +++ b/frontend/src/shell/database-management/DatabaseGrid.tsx @@ -1,4 +1,4 @@ -import { useEffect, useMemo, useRef, useState, type ReactNode, type RefObject } from "react"; +import { useEffect, useMemo, useRef, useState, type RefObject } from "react"; import { Menu } from "@base-ui/react/menu"; import { AgGridReact } from "ag-grid-react"; import { @@ -9,7 +9,7 @@ import { } from "ag-grid-community"; import "ag-grid-community/styles/ag-grid.css"; import "ag-grid-community/styles/ag-theme-alpine.css"; -import { ChevronDown, Eye, History, Info, Pencil, RefreshCw, Rows3, Sparkles, Trash2, X } from "lucide-react"; +import { ChevronDown, Eye, History, Info, Link2, Pencil, RefreshCw, Rows3, Sparkles, Trash2, X } from "lucide-react"; import { Button } from "../../components/ui/button"; import type { CatalogDatabase, @@ -17,9 +17,9 @@ import type { CatalogSyncScope, DescriptionGenerationScope, } from "../../api/catalog-databases"; -import { statusLabel } from "./model"; import { databaseSyncItemClass, databaseSyncScopes } from "./DatabaseSyncMenu"; import { FleetActionSelector, type FleetActionOption } from "./FleetActionSelector"; +import { NO_METADATA_GENERATION_LLM_MODEL_MESSAGE } from "./MetadataGenerationModelSelector"; ModuleRegistry.registerModules([AllCommunityModule]); @@ -33,6 +33,7 @@ interface DatabaseGridProps { onSearchChange: (value: string) => void; onView: (row: CatalogDatabase, origin: HTMLButtonElement) => void; onOpenTables?: (row: CatalogDatabase, origin: HTMLButtonElement) => void; + onOpenRelationships?: (row: CatalogDatabase, origin: HTMLButtonElement) => void; onEdit: (row: CatalogDatabase, origin: HTMLButtonElement) => void; onDelete: (row: CatalogDatabase, origin: HTMLButtonElement) => void; onOpenSync: (row: CatalogDatabase) => void; @@ -49,7 +50,6 @@ interface DatabaseGridProps { rows: CatalogDatabase[], target: CatalogDatabaseMetadataDeleteTarget, ) => Promise; - metadataModelControl?: ReactNode; onRefresh?: () => void | Promise; presentation?: "legacy" | "fleet"; } @@ -59,6 +59,7 @@ interface DatabaseGridContext { presentation: "legacy" | "fleet"; onView: DatabaseGridProps["onView"]; onOpenTables?: DatabaseGridProps["onOpenTables"]; + onOpenRelationships?: DatabaseGridProps["onOpenRelationships"]; onEdit: DatabaseGridProps["onEdit"]; onDelete: DatabaseGridProps["onDelete"]; onOpenSync: DatabaseGridProps["onOpenSync"]; @@ -91,18 +92,120 @@ function useCompactViewport(): boolean { return compact; } -function StatusPill({ row }: { row: CatalogDatabase }) { - const tone = !row.workspaceAvailable - ? "border-destructive/30 bg-destructive/10 text-destructive" - : !row.configured - ? "border-border bg-muted text-muted-foreground" - : row.connectionStatus === "reachable" - ? "border-emerald-300/70 bg-emerald-50 text-emerald-800 dark:bg-emerald-950/30 dark:text-emerald-300" - : row.connectionStatus === "failed" - ? "border-destructive/30 bg-destructive/10 text-destructive" - : "border-amber-300/70 bg-amber-50 text-amber-900 dark:bg-amber-950/30 dark:text-amber-300"; +type StateTone = "neutral" | "info" | "success" | "warning" | "danger"; - return
{statusLabel(row)}{row.activeSyncRun ? Syncing : null}
; +const stateToneClass: Record = { + neutral: "bg-muted-foreground/55", + info: "bg-sky-500", + success: "bg-emerald-500", + warning: "bg-amber-500", + danger: "bg-destructive", +}; + +function StateCell({ + label, + detail, + tone, + detailClassName = "text-muted-foreground", +}: { + label: string; + detail: string; + tone: StateTone; + detailClassName?: string; +}) { + return ( +
+ + + {detail} +
+ ); +} + +function evidenceStatus(row: CatalogDatabase): { label: string; tone: StateTone } { + if (!row.workspaceEvidence) { + return { label: "Evidence state unavailable", tone: "warning" }; + } + switch (row.workspaceEvidence.state) { + case "materialized_current_revision": + return { label: "Evidence materialized", tone: "success" }; + case "configuration_required": + return { label: "Evidence credentials required", tone: "warning" }; + case "configured_unverified": + return { label: "Evidence configured, unverified", tone: "info" }; + case "workspace_unavailable": + return { label: "Evidence unavailable", tone: "danger" }; + default: + return { label: "No Evidence declared", tone: "neutral" }; + } +} + +function RevisionEvidenceCell({ row }: { row: CatalogDatabase }) { + if (!row.workspaceRevision) { + return ; + } + const evidence = evidenceStatus(row); + const source = row.workspaceEvidence?.sourceType ? ` · ${row.workspaceEvidence.sourceType}` : ""; + return ( + + ); +} + +function transportLabel(row: CatalogDatabase): string { + const transport = row.runtimeBinding?.transport; + if (transport === "postgres_direct") return "Direct PostgreSQL"; + if (transport === "rest_api") return "REST API"; + if (transport === "ssh_tunnel") return "SSH tunnel"; + return "No runtime binding"; +} + +function RuntimeBindingCell({ row }: { row: CatalogDatabase }) { + if (!row.runtimeBinding) { + return ; + } + if (!row.runtimeBinding.sessionTransportSupported) { + return ; + } + if (row.runtimeBinding.configurationState === "configuration_required") { + return ; + } + return ; +} + +function catalogConnectionDetail(row: CatalogDatabase): { label: string; className?: string } { + if (row.activeSyncRun) return { label: "Metadata sync in progress", className: "text-primary" }; + if (row.testedVersion !== undefined && row.testedVersion !== row.version) { + return { label: "Connection retest required", className: "text-amber-700 dark:text-amber-300" }; + } + if (row.connectionStatus === "reachable") return { label: "Connection reachable" }; + if (row.connectionStatus === "failed") { + return { label: "Connection test failed", className: "text-destructive" }; + } + return { label: "Connection not tested" }; +} + +function MetadataCatalogCell({ row }: { row: CatalogDatabase }) { + if (!row.workspaceAvailable) { + return ; + } + if (!row.configured) { + return ; + } + const connection = catalogConnectionDetail(row); + return ( + + ); } function DatabaseActionsCell({ @@ -111,11 +214,48 @@ function DatabaseActionsCell({ }: ICellRendererParams) { if (!data || !context) return null; + if (data.workspaceAvailable && !data.configured) { + const configureReasonId = `database-configure-reason-${data.workspaceId}`; + return ( +
event.stopPropagation()}> + + + {!context.canManage ? ( + + Database management permission is required. + + ) : null} +
+ ); + } + const editDisabled = !context.canManage || !data.workspaceAvailable; const deleteDisabled = !context.canManage || !data.configured; const editLabel = `Edit ${data.workspaceName}`; const editReasonId = `database-edit-reason-${data.workspaceId}`; const deleteReasonId = `database-delete-reason-${data.workspaceId}`; + const relationshipsReasonId = `database-relationships-reason-${data.workspaceId}`; return (
event.stopPropagation()}> @@ -138,6 +278,26 @@ function DatabaseActionsCell({ ) : null} + {context.presentation === "fleet" && context.onOpenRelationships ? ( + + + {!data.configured || !data.id ? ( + + Save this database configuration before managing relationships. + + ) : null} + + ) : null} ) : null}} - {configuredCount}/{rows.length} + {configuredCount}/{rows.length} catalogs configured {isFetching && !isLoading ? ( Refreshing @@ -660,7 +835,7 @@ export function DatabaseGrid({ if (pendingGenerationScope && action === null) setPendingGenerationScope(null); setSelectedRows(api.getSelectedRows()); }} - rowHeight={44} + rowHeight={52} headerHeight={38} animateRows={false} /> diff --git a/frontend/src/shell/database-management/DatabaseRelationships.test.tsx b/frontend/src/shell/database-management/DatabaseRelationships.test.tsx new file mode 100644 index 00000000..dcb46dde --- /dev/null +++ b/frontend/src/shell/database-management/DatabaseRelationships.test.tsx @@ -0,0 +1,379 @@ +import { render, screen, waitFor, within } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { http, HttpResponse } from "msw"; +import { expect, test, vi } from "vitest"; + +import type { + CatalogColumn, + CatalogDatabase, + CatalogRelationship, + CatalogTable, +} from "../../api/catalog-databases"; +import { Toaster } from "../../components/ui/sonner"; +import { server } from "../../test/msw"; +import { DatabaseRelationships } from "./DatabaseRelationships"; + +const databaseId = "11111111-1111-4111-8111-111111111111"; + +const database: CatalogDatabase = { + id: databaseId, + workspaceId: "psd", + workspaceName: "Policlinico San Donato", + workspaceAvailable: true, + workspaceRevision: { commit: "a".repeat(40), blob: "b".repeat(40) }, + workspaceEvidence: { sourceType: "filesystem", state: "materialized_current_revision" }, + runtimeBinding: { transport: "postgres_direct", configurationState: "ready", sessionTransportSupported: true }, + configured: true, + engine: "postgres", + databaseName: "analytics", + schema: "public", + binding: { transport: "postgres_direct", port: 5432 }, + connectionStatus: "reachable", + testedVersion: 3, + version: 3, + createdAt: "2026-08-31T12:00:00.000Z", + updatedAt: "2026-08-31T12:00:00.000Z", + activeSyncRun: undefined, + secrets: { + password: false, + apiKey: false, + sshPrivateKey: false, + sshPrivateKeyPassphrase: false, + sshKnownHosts: false, + tlsCa: false, + }, +}; + +function relationship( + id: string, + origin: CatalogRelationship["origin"], + status: CatalogRelationship["status"], + sourceTableName: string, +): CatalogRelationship { + const physical = origin === "physical"; + return { + id, + databaseId, + constraintName: physical ? `${sourceTableName}_user_id_fkey` : null, + sourceTableId: `${id}-source-table`, + sourceTableName, + targetTableId: `${id}-target-table`, + targetTableName: "users", + updateRule: physical ? "NO ACTION" : null, + deleteRule: physical ? "CASCADE" : null, + deferrable: false, + initiallyDeferred: false, + columns: [{ + position: 1, + sourceColumnId: `${id}-source-column`, + sourceColumnName: "user_id", + targetColumnId: `${id}-target-column`, + targetColumnName: "id", + }], + origin, + status, + lastSyncedDatabaseVersion: physical ? 3 : null, + lastSyncedAt: physical ? "2026-08-31T12:00:00.000Z" : null, + createdAt: "2026-08-31T12:00:00.000Z", + updatedAt: "2026-08-31T12:00:00.000Z", + }; +} + +function renderRelationships(rows: CatalogRelationship[], canManage = true) { + server.use(http.get( + "/api/catalog/databases/:databaseId/relationships", + () => HttpResponse.json(rows), + )); + const client = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + const callbacks = { + onBackToDatabases: vi.fn(), + onOpenOverview: vi.fn(), + onOpenTables: vi.fn(), + onRunStarted: vi.fn(), + onOpenSync: vi.fn(), + }; + const view = render( + + + + , + ); + return { ...view, client, callbacks }; +} + +test("shows one relationship map and filters active, excluded, and all relationships", async () => { + const user = userEvent.setup(); + renderRelationships([ + relationship("physical", "physical", "active", "visits"), + relationship("generated", "generated", "active", "orders"), + relationship("manual", "manual", "active", "payments"), + relationship("excluded", "generated", "excluded", "legacy_orders"), + ]); + + expect(await screen.findByText("visits")).toBeVisible(); + expect(screen.getByText("orders")).toBeVisible(); + expect(screen.getByText("payments")).toBeVisible(); + expect(screen.queryByText("legacy_orders")).not.toBeInTheDocument(); + expect(screen.getByText("Physical")).toBeVisible(); + expect(screen.getByText("Generated")).toBeVisible(); + expect(screen.getByText("Manual")).toBeVisible(); + + await user.selectOptions(screen.getByRole("combobox", { name: "Relationship status" }), "excluded"); + expect(await screen.findByText("legacy_orders")).toBeVisible(); + await waitFor(() => expect(screen.queryByText("visits")).not.toBeInTheDocument()); + + await user.selectOptions(screen.getByRole("combobox", { name: "Relationship status" }), "all"); + expect(await screen.findByText("visits")).toBeVisible(); + expect(screen.getByText("legacy_orders")).toBeVisible(); +}); + +test("adds a manual relationship from four catalog selections", async () => { + const user = userEvent.setup(); + const orders: CatalogTable = { + id: "22222222-2222-4222-8222-222222222222", + databaseId, + name: "orders", + sourceComment: null, + description: null, + generatedDescription: null, + version: 1, + createdAt: "2026-08-31T12:00:00.000Z", + updatedAt: "2026-08-31T12:00:00.000Z", + }; + const users: CatalogTable = { + ...orders, + id: "33333333-3333-4333-8333-333333333333", + name: "users", + }; + const sourceColumn: CatalogColumn = { + id: "44444444-4444-4444-8444-444444444444", + tableId: orders.id, + name: "user_id", + ordinalPosition: 1, + dataType: "bigint", + isNullable: false, + defaultExpression: null, + primaryKeyPosition: null, + isPrimaryKey: false, + isForeignKey: false, + foreignKeyCount: 0, + sourceComment: null, + description: null, + generatedDescription: null, + sensitive: false, + lastSyncedDatabaseVersion: 3, + lastSyncedAt: "2026-08-31T12:00:00.000Z", + version: 1, + createdAt: "2026-08-31T12:00:00.000Z", + updatedAt: "2026-08-31T12:00:00.000Z", + }; + const targetColumn: CatalogColumn = { + ...sourceColumn, + id: "55555555-5555-4555-8555-555555555555", + tableId: users.id, + name: "id", + primaryKeyPosition: 1, + isPrimaryKey: true, + }; + let requestBody: unknown; + server.use( + http.get("/api/catalog/databases/:databaseId/tables", () => HttpResponse.json([orders, users])), + http.get( + "/api/catalog/databases/:databaseId/tables/:tableId/columns", + ({ params }) => HttpResponse.json(params.tableId === orders.id ? [sourceColumn] : [targetColumn]), + ), + http.post("/api/catalog/databases/:databaseId/relationships", async ({ request }) => { + requestBody = await request.json(); + return HttpResponse.json(relationship("created", "manual", "active", "orders"), { status: 201 }); + }), + ); + renderRelationships([]); + + await user.click(await screen.findByRole("button", { name: "Add relationship" })); + const drawer = await screen.findByRole("dialog", { name: "Add relationship" }); + expect(drawer).toHaveAttribute("aria-modal", "false"); + const submit = within(drawer).getByRole("button", { name: "Add relationship" }); + + await user.selectOptions(screen.getByRole("combobox", { name: "Source table" }), orders.id); + await user.selectOptions(screen.getByRole("combobox", { name: "Source column" }), sourceColumn.id); + await user.selectOptions(screen.getByRole("combobox", { name: "Target table" }), users.id); + await user.selectOptions(screen.getByRole("combobox", { name: "Target column" }), targetColumn.id); + expect(submit).toBeEnabled(); + await user.click(submit); + + await waitFor(() => expect(requestBody).toEqual({ + sourceColumnId: sourceColumn.id, + targetColumnId: targetColumn.id, + })); + expect(await screen.findByText("Relationship added.")).toBeVisible(); + await waitFor(() => expect(screen.queryByRole("dialog", { name: "Add relationship" })).not.toBeInTheDocument()); +}); + +test("rebuilds generated relationships with progress and a result summary", async () => { + const user = userEvent.setup(); + let finish: (() => void) | undefined; + server.use(http.post( + "/api/catalog/databases/:databaseId/relationships/rebuild-generated", + async () => { + await new Promise((resolve) => { finish = resolve; }); + return HttpResponse.json({ added: 12, alreadyPresent: 7, excluded: 3, ambiguous: 2 }); + }, + )); + renderRelationships([]); + + await user.selectOptions( + await screen.findByRole("combobox", { name: "Relationship action" }), + "rebuild-generated", + ); + await user.click(screen.getByRole("button", { name: "Run action" })); + expect(await screen.findByRole("button", { name: "Rebuilding…" })).toBeDisabled(); + + finish?.(); + expect(await screen.findByText( + "Rebuild complete: 12 added, 7 already present, 3 excluded, 2 ambiguous.", + )).toBeVisible(); +}); + +test("reports when rebuilding finds no new relationships", async () => { + const user = userEvent.setup(); + server.use(http.post( + "/api/catalog/databases/:databaseId/relationships/rebuild-generated", + () => HttpResponse.json({ added: 0, alreadyPresent: 0, excluded: 0, ambiguous: 0 }), + )); + renderRelationships([]); + + await user.selectOptions( + await screen.findByRole("combobox", { name: "Relationship action" }), + "rebuild-generated", + ); + await user.click(screen.getByRole("button", { name: "Run action" })); + expect(await screen.findByText("Rebuild complete: no new relationships found.")).toBeVisible(); +}); + +test("excludes a generated relationship after an explanatory drawer confirmation", async () => { + const user = userEvent.setup(); + const generated = relationship("generated", "generated", "active", "orders"); + let requestBody: unknown; + server.use(http.patch( + "/api/catalog/databases/:databaseId/relationships/:relationshipId", + async ({ request }) => { + requestBody = await request.json(); + return HttpResponse.json({ ...generated, status: "excluded" }); + }, + )); + renderRelationships([generated]); + + await user.click(await screen.findByRole("button", { + name: "View relationship orders.user_id to users.id", + })); + const drawer = await screen.findByRole("dialog", { name: "Relationship details" }); + expect(within(drawer).getByText("Generated")).toBeVisible(); + await user.click(within(drawer).getByRole("button", { name: "Exclude" })); + expect(within(drawer).getByText( + "The relationship will move to Excluded. Rebuilds will not recreate it, and you can restore it later.", + )).toBeVisible(); + const confirmExclude = within(drawer).getByRole("button", { name: "Exclude relationship" }); + await waitFor(() => expect(confirmExclude).toHaveFocus()); + await user.click(confirmExclude); + + await waitFor(() => expect(requestBody).toEqual({ status: "excluded" })); + expect(await screen.findByText( + "Relationship excluded. Future rebuilds will leave it excluded.", + )).toBeVisible(); +}); + +test("keeps physical relationships read-only in the details drawer", async () => { + const user = userEvent.setup(); + renderRelationships([relationship("physical", "physical", "active", "visits")]); + + await user.click(await screen.findByRole("button", { + name: "View relationship visits.user_id to users.id", + })); + const drawer = await screen.findByRole("dialog", { name: "Relationship details" }); + expect(within(drawer).getByText("This information is synchronized from the source schema and is read-only.")).toBeVisible(); + expect(within(drawer).queryByRole("button", { name: "Exclude" })).not.toBeInTheDocument(); + expect(within(drawer).queryByRole("button", { name: "Restore" })).not.toBeInTheDocument(); + expect(within(drawer).queryByRole("button", { name: "Delete permanently" })).not.toBeInTheDocument(); +}); + +test("permanently deletes a logical relationship after warning that rebuild may recreate it", async () => { + const user = userEvent.setup(); + const manual = relationship("manual", "manual", "active", "payments"); + let deletes = 0; + server.use(http.delete( + "/api/catalog/databases/:databaseId/relationships/:relationshipId", + () => { + deletes += 1; + return new HttpResponse(null, { status: 204 }); + }, + )); + renderRelationships([manual]); + + await user.click(await screen.findByRole("button", { + name: "View relationship payments.user_id to users.id", + })); + const drawer = await screen.findByRole("dialog", { name: "Relationship details" }); + await user.click(within(drawer).getByRole("button", { name: "Delete permanently" })); + expect(within(drawer).getByText( + "The relationship record will be erased. A future rebuild may infer it again.", + )).toBeVisible(); + await user.click(within(drawer).getByRole("button", { name: "Delete permanently" })); + + await waitFor(() => expect(deletes).toBe(1)); + expect(await screen.findByText( + "Relationship deleted permanently. A future rebuild may infer it again.", + )).toBeVisible(); +}); + +test("restores an excluded logical relationship", async () => { + const user = userEvent.setup(); + const excluded = relationship("excluded", "generated", "excluded", "legacy_orders"); + let requestBody: unknown; + server.use(http.patch( + "/api/catalog/databases/:databaseId/relationships/:relationshipId", + async ({ request }) => { + requestBody = await request.json(); + return HttpResponse.json({ ...excluded, status: "active" }); + }, + )); + renderRelationships([excluded]); + + await user.selectOptions( + await screen.findByRole("combobox", { name: "Relationship status" }), + "excluded", + ); + await user.click(await screen.findByRole("button", { + name: "View relationship legacy_orders.user_id to users.id", + })); + const drawer = await screen.findByRole("dialog", { name: "Relationship details" }); + await user.click(within(drawer).getByRole("button", { name: "Restore" })); + + await waitFor(() => expect(requestBody).toEqual({ status: "active" })); + expect(await screen.findByText("Relationship restored.")).toBeVisible(); +}); + +test("keeps relationship context open when a logical mutation fails", async () => { + const user = userEvent.setup(); + const generated = relationship("generated-error", "generated", "active", "orders"); + server.use(http.patch( + "/api/catalog/databases/:databaseId/relationships/:relationshipId", + () => HttpResponse.json({ code: "relationship_not_found" }, { status: 404 }), + )); + renderRelationships([generated]); + + await user.click(await screen.findByRole("button", { + name: "View relationship orders.user_id to users.id", + })); + const drawer = await screen.findByRole("dialog", { name: "Relationship details" }); + await user.click(within(drawer).getByRole("button", { name: "Exclude" })); + await user.click(within(drawer).getByRole("button", { name: "Exclude relationship" })); + + expect(await screen.findByText("The relationship no longer exists. Refresh and try again.")).toBeVisible(); + expect(screen.getByRole("dialog", { name: "Relationship details" })).toBeVisible(); +}); diff --git a/frontend/src/shell/database-management/DatabaseRelationships.tsx b/frontend/src/shell/database-management/DatabaseRelationships.tsx index 5163a22e..777c7e5e 100644 --- a/frontend/src/shell/database-management/DatabaseRelationships.tsx +++ b/frontend/src/shell/database-management/DatabaseRelationships.tsx @@ -1,19 +1,26 @@ -import { useMemo, useRef, useState, type ReactNode } from "react"; -import { useQuery } from "@tanstack/react-query"; +import { useEffect, useMemo, useRef, useState } from "react"; +import { useQuery, useQueryClient } from "@tanstack/react-query"; import { AgGridReact } from "ag-grid-react"; import type { ColDef, ICellRendererParams } from "ag-grid-community"; -import { ArrowLeft, History, Info, Play, RefreshCw } from "lucide-react"; +import { ArrowLeft, History, Info, Play, Plus, RefreshCw } from "lucide-react"; import { toast } from "sonner"; import { Button } from "../../components/ui/button"; -import { apiErrorMessage } from "../../api/client"; +import { ApiError, apiErrorMessage } from "../../api/client"; import { + createCatalogRelationship, + deleteCatalogRelationship, listCatalogRelationships, + listCatalogColumns, + listCatalogTables, + rebuildGeneratedRelationships, + setCatalogRelationshipStatus, startCatalogSync, type CatalogDatabase, type CatalogRelationship, type CatalogSyncRun, } from "../../api/catalog-databases"; import { FleetLedgerDrawer } from "./FleetLedgerShell"; +import type { DatabaseNavigationState } from "./model"; interface Props { database: CatalogDatabase; @@ -23,7 +30,10 @@ interface Props { onOpenTables: () => void; onRunStarted: (run: CatalogSyncRun) => void; onOpenSync: () => void; - metadataModelControl?: ReactNode; + onOpenDescriptionHistory?: () => void; + onOpenSensitiveHistory?: () => void; + onCatalogMetricsChanged?: () => void | Promise; + onNavigationStateChange?: (state: DatabaseNavigationState) => void; presentation?: "legacy" | "fleet"; } @@ -32,6 +42,19 @@ interface RelationshipGridContext { onView: (relationship: CatalogRelationship, origin: HTMLButtonElement) => void; } +type RelationshipStatusFilter = "active" | "excluded" | "all"; + +function titleCase(value: string): string { + return value.charAt(0).toUpperCase() + value.slice(1); +} + +function relationshipLabel(relationship: CatalogRelationship): string { + const pair = relationship.columns[0]; + const source = `${relationship.sourceTableName}${pair ? `.${pair.sourceColumnName}` : ""}`; + const target = `${relationship.targetTableName}${pair ? `.${pair.targetColumnName}` : ""}`; + return `${source} to ${target}`; +} + function RelationshipActionsCell({ data, context }: ICellRendererParams) { if (!data || !context || context.presentation !== "fleet") return null; return ( @@ -42,7 +65,7 @@ function RelationshipActionsCell({ data, context }: ICellRendererParams context.onView(data, event.currentTarget)} >
: null}
- Physical relationships + Relationship map + + setSearch(event.target.value)} /> - {data.length} + {visibleRelationships.length} {presentation === "fleet" ? (
- {selectedAction === "all" ? "This database" : selectedAction === "relationships" ? "All relationships in this database" : "This database"} + {selectedAction === "relationships" ? "Physical relationships in this database" : "This database"} Action scope
@@ -173,18 +381,37 @@ export function DatabaseRelationships({ className="thot-fleet-action-selector__select" value={selectedAction} disabled={busy} - onChange={(event) => setSelectedAction(event.target.value as "" | "relationships" | "all")} + onChange={(event) => setSelectedAction(event.target.value as "" | "rebuild-generated" | "relationships" | "all")} > + + + - + - + + {onOpenDescriptionHistory ? : null} + {onOpenSensitiveHistory ? : null} + + {addUnavailable ? {addUnavailable} : null} {selectedActionUnavailable ? {selectedActionUnavailable} : null}
) : } @@ -192,26 +419,174 @@ export function DatabaseRelationships({ {!bindingReady ?
{presentation === "fleet" ? "Test the current database binding in database details before synchronizing relationships." : "Test the current database binding from Overview before synchronizing relationships."}
: null} {fleetQueryError ??
- rowData={data} columnDefs={columns} context={context} loading={isLoading} quickFilterText={search} defaultColDef={{ sortable: true, filter: true, resizable: true }} getRowId={({ data: row }) => row.id} rowHeight={44} headerHeight={38} animateRows={false} overlayNoRowsTemplate="No physical relationships synchronized for this database." /> + rowData={visibleRelationships} columnDefs={columns} context={context} loading={isLoading} quickFilterText={search} defaultColDef={{ sortable: true, filter: true, resizable: true }} getRowId={({ data: row }) => row.id} rowHeight={44} headerHeight={38} animateRows={false} overlayNoRowsTemplate={statusFilter === "active" ? "No relationships yet. Rebuild from column names or add one manually." : statusFilter === "excluded" ? "No excluded relationships." : "No relationships for this database."} />
}
+ {presentation === "fleet" && addOpen ? ( + + + + + )} + > +
+ + + + + {tablesQuery.isError || sourceColumnsQuery.isError || targetColumnsQuery.isError ? ( +

Catalog tables or columns could not be loaded.

+ ) : null} + {addError ?

{addError}

: null} +
+ + ) : null} {presentation === "fleet" && activeRelationship ? ( Close} + busy={busy} + footer={pendingMutation ? ( +
{ + if (event.key === "Escape" && !busy) setPendingMutation(null); + }} + > +

+ {pendingMutation === "exclude" + ? "The relationship will move to Excluded. Rebuilds will not recreate it, and you can restore it later." + : "The relationship record will be erased. A future rebuild may infer it again."} +

+ + +
+ ) : ( + <> + + {activeRelationship.origin !== "physical" && activeRelationship.status === "active" ? ( + + ) : null} + {activeRelationship.origin !== "physical" && activeRelationship.status === "excluded" ? ( + + ) : null} + {activeRelationship.origin !== "physical" ? ( + + ) : null} + + )} >
-
Constraint
{activeRelationship.constraintName}
+
Origin
{titleCase(activeRelationship.origin)}
+
Status
{titleCase(activeRelationship.status)}
+ {activeRelationship.constraintName ?
Constraint
{activeRelationship.constraintName}
: null}
Source table
{activeRelationship.sourceTableName}
Target table
{activeRelationship.targetTableName}
-
On update
{activeRelationship.updateRule ?? "Not specified"}
-
On delete
{activeRelationship.deleteRule ?? "Not specified"}
+ {activeRelationship.origin === "physical" ? <>
On update
{activeRelationship.updateRule ?? "Not specified"}
On delete
{activeRelationship.deleteRule ?? "Not specified"}
: null}
Column mapping
diff --git a/frontend/src/shell/database-management/DatabaseTables.tsx b/frontend/src/shell/database-management/DatabaseTables.tsx index e7fb0a4a..0112c399 100644 --- a/frontend/src/shell/database-management/DatabaseTables.tsx +++ b/frontend/src/shell/database-management/DatabaseTables.tsx @@ -1,4 +1,4 @@ -import { useEffect, useMemo, useRef, useState, type ReactNode } from "react"; +import { useEffect, useMemo, useRef, useState } from "react"; import { Menu } from "@base-ui/react/menu"; import { useQuery, useQueryClient } from "@tanstack/react-query"; import { AgGridReact } from "ag-grid-react"; @@ -25,6 +25,7 @@ import type { DatabaseNavigationState } from "./model"; import { DatabaseColumns } from "./DatabaseColumns"; import { FleetActionSelector, type FleetActionOption } from "./FleetActionSelector"; import { FleetLedgerDrawer } from "./FleetLedgerShell"; +import { NO_METADATA_GENERATION_LLM_MODEL_MESSAGE } from "./MetadataGenerationModelSelector"; interface Props { database: CatalogDatabase; @@ -39,10 +40,11 @@ interface Props { onNestedNavigationChange?: (active: boolean) => void; onRunStarted: (run: CatalogSyncRun) => void; onOpenSync: () => void; + onOpenDescriptionHistory?: () => void; + onOpenSensitiveHistory?: () => void; onDescriptionGenerationRunStarted: (run: DescriptionGenerationRun) => void; onSuggestSensitive: (selection: SensitiveDataSuggestionRequest, scopeLabel: string) => Promise; onCatalogMetricsChanged?: () => void | Promise; - metadataModelControl?: ReactNode; presentation?: "legacy" | "fleet"; } @@ -93,10 +95,11 @@ export function DatabaseTables({ onNestedNavigationChange, onRunStarted, onOpenSync, + onOpenDescriptionHistory, + onOpenSensitiveHistory, onDescriptionGenerationRunStarted, onSuggestSensitive, onCatalogMetricsChanged, - metadataModelControl, presentation = "legacy", }: Props) { const databaseId = database.id!; @@ -314,7 +317,7 @@ export function DatabaseTables({ : selectedIds.length === 0 ? "Select at least one table." : !selectedMetadataModel - ? "Choose a metadata model first." + ? NO_METADATA_GENERATION_LLM_MODEL_MESSAGE : descriptionGenerationActive || Boolean(currentRun) ? "Wait for the active catalog operation to finish." : busy !== null @@ -377,7 +380,7 @@ export function DatabaseTables({ : selectedIds.length === 0 ? "Select at least one table." : !selectedMetadataModel - ? "Choose a metadata model first." + ? NO_METADATA_GENERATION_LLM_MODEL_MESSAGE : descriptionGenerationActive || Boolean(currentRun) ? "Wait for the active catalog operation to finish." : busy !== null @@ -487,8 +490,9 @@ export function DatabaseTables({
) : null} + {onOpenDescriptionHistory ? : null} + {onOpenSensitiveHistory ? : null}
{!bindingReady ?
Test the current database binding from Overview before synchronizing tables.
: null} {fleetQueryError ??
diff --git a/frontend/src/shell/database-management/DescriptionGenerationDrawer.test.tsx b/frontend/src/shell/database-management/DescriptionGenerationDrawer.test.tsx index 9a6e36c3..44c957d1 100644 --- a/frontend/src/shell/database-management/DescriptionGenerationDrawer.test.tsx +++ b/frontend/src/shell/database-management/DescriptionGenerationDrawer.test.tsx @@ -248,6 +248,33 @@ test("reopens a terminal run from newest-first persisted history", async () => { expect(within(drawer).getByText("3", { selector: "[data-count='generated']" })).toBeVisible(); }); +test("styles a successfully completed run with the success background", async () => { + const completedRun: DescriptionGenerationRun = { + ...runningRun, + status: "completed", + total: 1, + processed: 1, + generated: 1, + updatedAt: "2026-08-28T08:02:00Z", + finishedAt: "2026-08-28T08:02:00Z", + }; + server.use( + http.get("/api/catalog/description-generation-runs", () => HttpResponse.json([completedRun])), + http.get("/api/catalog/description-generation-runs/:runId", () => HttpResponse.json(completedRun)), + http.get("/api/catalog/description-generation-runs/:runId/events-list", () => HttpResponse.json([])), + ); + renderDrawer({ initialRun: completedRun }); + + const history = await screen.findByRole("region", { + name: "Description generation history", + }); + const completedHistoryItem = within(history).getByRole("button", { + name: /Completed.*missing/i, + }); + expect(completedHistoryItem).toHaveAttribute("data-status", "completed"); + expect(completedHistoryItem).toHaveClass("bg-[oklch(var(--success)/0.1)]"); +}); + test("offers Unlock only for an apparently stale active run and explains the interruption", async () => { const user = userEvent.setup(); const staleRun: DescriptionGenerationRun = { diff --git a/frontend/src/shell/database-management/DescriptionGenerationDrawer.tsx b/frontend/src/shell/database-management/DescriptionGenerationDrawer.tsx index 486e02ca..45acc36a 100644 --- a/frontend/src/shell/database-management/DescriptionGenerationDrawer.tsx +++ b/frontend/src/shell/database-management/DescriptionGenerationDrawer.tsx @@ -1,4 +1,4 @@ -import { useCallback, useEffect, useRef, useState } from "react"; +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { useQuery, useQueryClient } from "@tanstack/react-query"; import { LoaderCircle, LockOpen, Square } from "lucide-react"; import { toast } from "sonner"; @@ -18,6 +18,7 @@ import { FleetLedgerDrawer } from "./FleetLedgerShell"; interface Props { open: boolean; + databaseId?: string | null; run: DescriptionGenerationRun | null; modelLabel: string; onClose: () => void; @@ -62,6 +63,7 @@ function timestamp(value: string | null): string { export function DescriptionGenerationDrawer({ open, + databaseId = null, run: initialRun, modelLabel, onClose, @@ -100,11 +102,15 @@ export function DescriptionGenerationDrawer({ ? 3_000 : false, }); + const visibleHistory = useMemo( + () => (historyQuery.data ?? []).filter((item) => !databaseId || item.databaseId === databaseId), + [databaseId, historyQuery.data], + ); useEffect(() => { - if (!open || initialRun || !historyQuery.data?.[0]) return; - onRunUpdate(historyQuery.data[0]); - }, [historyQuery.data, initialRun, onRunUpdate, open]); + if (!open || initialRun || !visibleHistory[0]) return; + onRunUpdate(visibleHistory[0]); + }, [initialRun, onRunUpdate, open, visibleHistory]); const mergeEvents = useCallback((incoming: DescriptionGenerationEvent[]) => { if (incoming.length === 0) return; @@ -406,7 +412,7 @@ export function DescriptionGenerationDrawer({

Recent runs

- {historyQuery.data?.length ?? 0} + {visibleHistory.length}
{historyQuery.isError ? ( @@ -415,17 +421,18 @@ export function DescriptionGenerationDrawer({

) : historyQuery.isLoading ? (

Loading run history…

- ) : (historyQuery.data ?? []).length === 0 ? ( + ) : visibleHistory.length === 0 ? (

No description generation runs yet.

) : (
- {(historyQuery.data ?? []).map((item) => ( + {visibleHistory.map((item) => (