From 79c4c925b5743fb0276c762b27c3542eda1995da Mon Sep 17 00:00:00 2001 From: Codex Date: Thu, 27 Aug 2026 22:43:54 +0200 Subject: [PATCH] feat: implement metadata catalog database management --- AGENTS.md | 14 + CONTEXT.md | 84 +- PROJECT_STATE.md | 52 +- README.md | 36 +- backend/package-lock.json | 2049 +++++++++++++++++ backend/package.json | 3 + backend/scripts/ssh-askpass.mjs | 6 + backend/src/app.ts | 52 + backend/src/auth/config.ts | 2 +- backend/src/auth/session-store.ts | 2 +- backend/src/auth/types.ts | 2 +- backend/src/catalog/memory-repository.ts | 692 ++++++ backend/src/catalog/migrate.ts | 51 + .../migrations/001_workspace_databases.ts | 58 + .../catalog/migrations/002_catalog_tables.ts | 27 + .../migrations/003_catalog_schema_sync.ts | 136 ++ ...004_catalog_runtime_sequence_privileges.ts | 29 + backend/src/catalog/operation-coordinator.ts | 28 + backend/src/catalog/postgres-access.ts | 258 +++ backend/src/catalog/repository.ts | 1052 +++++++++ backend/src/catalog/schema-introspector.ts | 498 ++++ backend/src/catalog/secrets.ts | 10 + backend/src/catalog/service.ts | 228 ++ backend/src/catalog/sync-worker.ts | 311 +++ backend/src/catalog/table-introspector.ts | 133 ++ backend/src/catalog/table-service.ts | 45 + backend/src/catalog/types.ts | 339 +++ backend/src/config.ts | 41 + backend/src/routes/catalog-databases.ts | 218 ++ backend/src/routes/catalog-schema.ts | 230 ++ backend/src/routes/catalog-tables.ts | 79 + backend/test/auth-config.test.ts | 1 + backend/test/auth-request-snapshot.test.ts | 2 +- backend/test/auth-routes-local.test.ts | 2 +- backend/test/auth.test.ts | 4 +- backend/test/authorization.test.ts | 4 +- backend/test/catalog-databases-routes.test.ts | 122 + backend/test/catalog-postgres-access.test.ts | 171 ++ .../catalog-repository.integration.test.ts | 125 + .../test/catalog-schema-introspector.test.ts | 194 ++ backend/test/catalog-schema-routes.test.ts | 224 ++ .../test/catalog-table-introspector.test.ts | 124 + backend/test/catalog-tables-routes.test.ts | 126 + backend/test/config.test.ts | 19 + compose.yaml | 58 + deploy/env/local.env.example | 3 + deploy/env/server.env.example | 3 + docker/catalog-db-init.sql | 25 + docker/core.Dockerfile | 2 + docs/adr/0001-postgres-metadata-catalog.md | 15 +- ...02-workspace-database-secret-references.md | 19 + ...03-installation-local-database-bindings.md | 26 + .../0004-fastify-kysely-metadata-catalog.md | 22 + ...e-catalog-tables-during-synchronization.md | 27 + ...rate-physical-and-logical-relationships.md | 8 + ...le-authoritative-schema-synchronization.md | 9 + docs/agents/domain.md | 7 + docs/agents/issue-tracker.md | 28 + docs/agents/triage-labels.md | 19 + docs/architecture/components.md | 9 +- docs/architecture/overview.md | 17 +- docs/contracts/catalog-schema-snapshot.md | 88 + docs/guida-utente.md | 6 +- ...026-08-26-metadata-catalog-from-thothai.md | 273 ++- frontend/src/api/catalog-databases.ts | 265 +++ frontend/src/api/client.ts | 17 + frontend/src/index.css | 49 + .../AppShell.database-management.test.tsx | 53 +- frontend/src/shell/AppShell.tsx | 66 +- .../src/shell/DatabaseManagementPage.test.tsx | 842 +++++++ frontend/src/shell/DatabaseManagementPage.tsx | 758 +++++- .../database-management/CatalogSyncDrawer.tsx | 402 ++++ .../database-management/DatabaseColumns.tsx | 192 ++ .../database-management/DatabaseForm.tsx | 510 ++++ .../database-management/DatabaseGrid.tsx | 329 +++ .../DatabaseRelationships.tsx | 103 + .../database-management/DatabaseSyncMenu.tsx | 54 + .../database-management/DatabaseTables.tsx | 315 +++ .../src/shell/database-management/model.ts | 120 + frontend/src/test/msw.ts | 3 +- scripts/run-stack.sh | 12 +- scripts/test-canonical-install-compose.sh | 2 +- scripts/test-default-compose.sh | 23 +- scripts/test-unified-compose.sh | 25 +- scripts/test-windows-clone-contract.ps1 | 8 +- scripts/verify-workspace-install-docs.sh | 6 +- 86 files changed, 12566 insertions(+), 135 deletions(-) create mode 100755 backend/scripts/ssh-askpass.mjs create mode 100644 backend/src/catalog/memory-repository.ts create mode 100644 backend/src/catalog/migrate.ts create mode 100644 backend/src/catalog/migrations/001_workspace_databases.ts create mode 100644 backend/src/catalog/migrations/002_catalog_tables.ts create mode 100644 backend/src/catalog/migrations/003_catalog_schema_sync.ts create mode 100644 backend/src/catalog/migrations/004_catalog_runtime_sequence_privileges.ts create mode 100644 backend/src/catalog/operation-coordinator.ts create mode 100644 backend/src/catalog/postgres-access.ts create mode 100644 backend/src/catalog/repository.ts create mode 100644 backend/src/catalog/schema-introspector.ts create mode 100644 backend/src/catalog/secrets.ts create mode 100644 backend/src/catalog/service.ts create mode 100644 backend/src/catalog/sync-worker.ts create mode 100644 backend/src/catalog/table-introspector.ts create mode 100644 backend/src/catalog/table-service.ts create mode 100644 backend/src/catalog/types.ts create mode 100644 backend/src/routes/catalog-databases.ts create mode 100644 backend/src/routes/catalog-schema.ts create mode 100644 backend/src/routes/catalog-tables.ts create mode 100644 backend/test/catalog-databases-routes.test.ts create mode 100644 backend/test/catalog-postgres-access.test.ts create mode 100644 backend/test/catalog-repository.integration.test.ts create mode 100644 backend/test/catalog-schema-introspector.test.ts create mode 100644 backend/test/catalog-schema-routes.test.ts create mode 100644 backend/test/catalog-table-introspector.test.ts create mode 100644 backend/test/catalog-tables-routes.test.ts create mode 100644 docker/catalog-db-init.sql create mode 100644 docs/adr/0002-workspace-database-secret-references.md create mode 100644 docs/adr/0003-installation-local-database-bindings.md create mode 100644 docs/adr/0004-fastify-kysely-metadata-catalog.md create mode 100644 docs/adr/0005-hard-delete-catalog-tables-during-synchronization.md create mode 100644 docs/adr/0006-separate-physical-and-logical-relationships.md create mode 100644 docs/adr/0007-durable-authoritative-schema-synchronization.md create mode 100644 docs/agents/domain.md create mode 100644 docs/agents/issue-tracker.md create mode 100644 docs/agents/triage-labels.md create mode 100644 docs/contracts/catalog-schema-snapshot.md create mode 100644 frontend/src/api/catalog-databases.ts create mode 100644 frontend/src/shell/DatabaseManagementPage.test.tsx create mode 100644 frontend/src/shell/database-management/CatalogSyncDrawer.tsx create mode 100644 frontend/src/shell/database-management/DatabaseColumns.tsx create mode 100644 frontend/src/shell/database-management/DatabaseForm.tsx create mode 100644 frontend/src/shell/database-management/DatabaseGrid.tsx create mode 100644 frontend/src/shell/database-management/DatabaseRelationships.tsx create mode 100644 frontend/src/shell/database-management/DatabaseSyncMenu.tsx create mode 100644 frontend/src/shell/database-management/DatabaseTables.tsx create mode 100644 frontend/src/shell/database-management/model.ts diff --git a/AGENTS.md b/AGENTS.md index bcb509f9..3e37dc82 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,5 +1,19 @@ # AGENTS.md +## Agent skills + +### Issue tracker + +Issues for this repository live in the self-hosted Gitea repository at `https://git.tylconsulting.it/mptyl/ThothII`; use its web UI or authenticated Gitea API. See `docs/agents/issue-tracker.md`. + +### Triage labels + +Use the canonical labels `needs-triage`, `needs-info`, `ready-for-agent`, `ready-for-human`, and `wontfix`. See `docs/agents/triage-labels.md`. + +### Domain docs + +This is a single-context repository with root `CONTEXT.md` and `docs/adr/`. See `docs/agents/domain.md`. + This file provides guidance to Codex (Codex.ai/code) when working with code in this repository. ## Start here diff --git a/CONTEXT.md b/CONTEXT.md index 484a1796..6a5aeb25 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -246,10 +246,23 @@ precedente o di un altro workspace. ## Catalogo dei metadati -**Workspace Database** — Il database associato in modo uno-a-uno a un workspace, -considerato nella sua interezza fisica: tutte le tabelle, le colonne e le relazioni -disponibili. La sua struttura fisica viene acquisita interrogando il database; il -Metadata Catalog non crea né possiede l'identità del workspace. +**Workspace Database** — Il database che appartiene a un solo workspace e non può essere +condiviso con altri workspace; un workspace può averne al massimo uno. È considerato nella +coppia composta dal database PostgreSQL e da un solo schema: tutte le tabelle, le colonne e +le relazioni catalogate appartengono a quello schema. Il Metadata Catalog conserva +l'associazione, ma non crea né possiede l'identità del workspace. + +**Database Binding** — La configurazione specifica di un'installazione che seleziona un +trasporto e fornisce i riferimenti necessari a raggiungere un Workspace Database. Non è una +seconda identità del database e non viene condivisa automaticamente fra installazioni. + +**Thoth REST Connector** — Il trasporto REST tipizzato con cui ThothII interroga ed +introspeziona un Workspace Database attraverso il contratto RPC DWH supportato. Non è un +client configurabile per API REST arbitrarie. + +**Orphaned Workspace Database** — Un Workspace Database il cui workspace non è più presente +nel catalogo autorevole. Rimane conservato per il recupero amministrativo, ma non può essere +usato dal workflow finché non viene riassegnato a un workspace esistente. **Metadata Catalog** — Il contesto amministrativo che raccoglie e cura i metadati di un Workspace Database. Non definisce quali elementi partecipano al workflow SQL. @@ -257,12 +270,61 @@ Workspace Database. Non definisce quali elementi partecipano al workflow SQL. **Database Profile** — L'insieme curato di scope, descrizioni e metadati semantici associato a un Workspace Database. -**Physical Schema Snapshot** — L'inventario della struttura fisica osservata in un -Workspace Database durante una specifica introspezione. Non è un progetto dello schema -né un'autorizzazione a modificarne la struttura. +**Physical Table** — Una tabella osservata nello schema esterno di un Workspace Database. +La sua identità e il suo nome appartengono al database esterno, non al Metadata Catalog. -**AI Proposal** — Un contenuto generato con l'ausilio dell'AI che non è ancora stato -approvato come contenuto canonico. +**Catalog Table** — La rappresentazione persistita di una Physical Table nel Metadata Catalog. +La sua appartenenza e identità fisica derivano esclusivamente dall'introspezione; soltanto i suoi +Catalog Metadata possono essere curati amministrativamente. +_Avoid_: SqlTable, managed table -**Publication** — Una versione approvata e immutabile dei contenuti del Metadata -Catalog resa disponibile ai suoi consumatori. +**Physical Column** — Una colonna osservata in una Physical Table, inclusi nome, posizione, +tipo e appartenenza a chiavi dichiarate. La sua identità e i suoi fatti strutturali appartengono +al database esterno. + +**Catalog Column** — La rappresentazione persistita di una Physical Column nel Metadata Catalog. +I fatti osservati sono governati dalla sincronizzazione; Description e Generated Description +sono metadati amministrativi modificabili. +_Avoid_: SqlColumn, managed column + +**Physical Relationship** — Un vincolo foreign key dichiarato nel database esterno. La sua +identità comprende il vincolo e la sequenza ordinata delle coppie di colonne che lo compongono. + +**Catalog Relationship** — La rappresentazione persistita di una Physical Relationship nel +Metadata Catalog. È governata esclusivamente dall'introspezione e non è creata o modificata +manualmente. +_Avoid_: denormalized FK, relationship string + +**Logical Relationship** — Una relazione semantica curata o inferita che non corrisponde +necessariamente a un vincolo fisico. Ha ownership e lifecycle distinti da Catalog Relationship. + +**Description** — Il testo curato e consolidato che descrive una Catalog Table o Catalog Column +per gli usi downstream. + +**Generated Description** — Una proposta modificabile sottoposta a revisione umana prima di +essere consolidata come Description. Rimane distinta dal commento osservato nel database. +_Avoid_: generated comment, source comment + +**Table Synchronization** — La riconciliazione esplicita che rende le Catalog Table di un +Workspace Database uguali alle Physical Table osservate: crea quelle nuove, aggiorna i metadati +di origine ed elimina definitivamente quelle assenti. Non modifica mai il database esterno. +_Avoid_: table import + +**Schema Synchronization** — La riconciliazione esplicita e autorevole di tabelle, colonne e +Catalog Relationship di un Workspace Database. Può operare su uno scope specifico oppure su +un unico snapshot completo tramite Synchronize All. + +**Catalog Sync Run** — L'esecuzione durevole in background di una Schema Synchronization, con +scope, stato, avanzamento e log propri. Al massimo un run per Workspace Database può essere attivo. + +**Catalog Freshness** — La corrispondenza fra uno scope sincronizzato e la versione corrente +della Database Binding. Uno scope rimane consultabile ma è stale finché non viene sincronizzato +con la binding corrente. + +**Catalog Metadata** — I campi mutabili che descrivono database, tabelle, colonne e relazioni, +distinti dai fatti strutturali governati dalla sincronizzazione. Possono essere popolati dall'AI, +da un'importazione o da una modifica amministrativa senza cambiare il database esterno. + +**Introspection Capability** — Una categoria di struttura fisica che una Database Binding +può osservare, come tabelle, colonne, relazioni, indici o enum. Una capability non disponibile +è distinta da una capability osservata che non ha restituito elementi. diff --git a/PROJECT_STATE.md b/PROJECT_STATE.md index 4b128b23..2c6e30e1 100644 --- a/PROJECT_STATE.md +++ b/PROJECT_STATE.md @@ -1,6 +1,6 @@ # ThothII — Project State -Last updated: 2026-08-26. +Last updated: 2026-08-27. This file is the short operational snapshot. Stable commands and the architecture mental model live in `AGENTS.md`; current design and runtime contracts live under `docs/architecture/`, @@ -16,8 +16,9 @@ frontend (React/SSE) → backend (Fastify) → pi --mode rpc → tht/harness → ``` The harness owns the deterministic eight-phase NL→SQL workflow and all session persistence. -The backend is a process/RPC/SSE bridge without a database of its own. The frontend renders the -review gates and keeps the live transcript in memory. See +The backend remains a process/RPC/SSE bridge for sessions and now also owns an isolated PostgreSQL +metadata catalog for administrative database configuration. The frontend renders the review gates +and keeps the live transcript in memory. See `docs/architecture/components.md` for the detailed component and data-flow map. ## Evidence restructuring — accepted @@ -62,6 +63,51 @@ Workspace descriptors use schema v3. For PSD, workspace content and runtime root separate uncommitted repository `/Users/mp/projects/tht-workspace-psd`. Secrets remain outside Git and are supplied only through installation-local protected files. +## Database management + +The database, table, and authoritative physical-schema catalog slices are implemented. Database +management opens a responsive AG Grid master-detail surface, lists every YAML workspace, creates +at most one PostgreSQL database configuration per workspace, edits direct PostgreSQL, REST API, or +SSH-tunnel installation bindings, replaces write-only encrypted secrets, and tests supported +connector bindings. + +Configured databases use pure hierarchical navigation through `Overview`, `Tables`, and +`Relationships`; a selected table has `Overview` and `Columns`. Physical membership, source +comments, column types/default/nullability/PK positions, and constraint-level ordered FK pairs are +immutable projections of the external schema. Curated and generated descriptions are editable; +generated descriptions start null and AI generation/consolidation is deferred. + +Schema refresh is one durable asynchronous engine with database-table, database-column, +selected-table-column, relationship, and full-database actions. Database-level menus expose the +table, all-column, relationship, and full scopes separately; selecting tables exposes column +synchronization for that subset. Runs have one-active-job-per-database exclusion, leases and +restart recovery, atomic apply, destructive-diff confirmation with re-scan, cancellation before +apply, retained history, and a live SSE log with polling fallback. Null metadata renders blank +rather than as a placeholder. + +Direct PostgreSQL and strict known-host-verified OpenSSH use `pg_catalog`. REST bindings use the +typed full-snapshot `POST /rpc/schema_snapshot` contract when available. Servers such as the +current PSD endpoint that exposes only `POST /rpc/run_query` use one catalog-owned read-only query +to return the exact same strict v1 snapshot in a single round trip. Both paths remain fail-closed: +an absent capability, query error, partial result, or invalid snapshot applies no catalog changes. +SSH is not yet enabled for NL→SQL session runtime. + +The catalog runs in the internal `catalog-db` PostgreSQL service. Kysely migrations are an explicit +one-shot `catalog-migrate` operation; `scripts/run-stack.sh` runs it before local startup. Runtime +sessions still consume the existing workspace configuration in this slice: database-management +records do not yet change the NL→SQL handoff. The accepted design is recorded in +`docs/plans/2026-08-26-metadata-catalog-from-thothai.md`, the snapshot contract under +`docs/contracts/`, and ADRs 0001–0007. + +Semantic aliases, value descriptions, synonyms, concepts, AI metadata generation/consolidation, +and logical relationships remain deferred to their dedicated slices. + +Integration of the completed metadata catalog with core schema-linking is explicitly deferred +until the database, table, column, relationship, and synchronization slices are complete. At that +point the next required design gate is to compare the catalog snapshot with the current DWH +preprocessing/schema-linking contracts and plan the cutover; this follow-up must not be treated as +optional cleanup or silently omitted. + ## Active deployment work and manual gates ### PSD server deployment program diff --git a/README.md b/README.md index 51b3a3ac..d18daa74 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,16 @@ # ThothII ThothII is a human-reviewed NL-to-SQL workflow with a React frontend and a Fastify/Pi/`tht` -core. The portable deployment runs exactly two application services; data services remain -external in this profile, except for the mandatory internal semantic services bundled in Compose. +core. The portable deployment runs two application services plus the installation-local metadata +catalog; DWH and LLM services remain external. Semantic services are bundled in Compose. Authentication is configured through the single host CLI tht: see the [local authentication guide](docs/install/authentication-local.md), [generic OIDC guide](docs/install/authentication-oidc.md), and [manual acceptance matrix](docs/testing/authentication-manual-acceptance.md). ## Docker Compose: local startup -Requirements: Docker Engine with Compose v2. The mandatory stack is `frontend`, `core`, `qdrant`, `embedding`, and the one-shot `embedding-model-init`. DWH and LLM remain external, +Requirements: Docker Engine with Compose v2. The mandatory stack is `frontend`, `core`, +`catalog-db`, `qdrant`, `embedding`, and the one-shot `embedding-model-init`. DWH and LLM remain external, configurable endpoints—even when they are co-located with ThothII. From a fresh clone, run these commands from the repository root: @@ -17,17 +18,28 @@ From a fresh clone, run these commands from the repository root: ```sh cp deploy/env/local.env.example deploy/env/local.env # Edit deploy/env/local.env, including PI_AUTH_FILE, THT_SECRETS_FILE, and external endpoints. -docker compose --env-file deploy/env/local.env \ - -f compose.yaml -f deploy/compose.local.yaml up --build -d +./scripts/run-stack.sh ``` -`./scripts/run-stack.sh` runs this same base+local command in the foreground. The core image -contains its Pi runtime; no host `pi` executable is used. For a server installation: +The launcher builds the core, starts `catalog-db`, runs the explicit one-shot Kysely migrations, +then runs the base+local stack in the foreground. Migrations never run implicitly in backend +startup. The core image contains its Pi runtime; no host `pi` executable is used. For a server +installation, build the image, start the catalog, and run the same migration service before the +application rollout: ```sh cp deploy/env/server.env.example deploy/env/server.env # Edit all absolute storage, Pi/secret/session files, and endpoint paths. sudo scripts/prepare-server-pi-state.sh /srv/thothii/pi-state 10001 10001 +docker compose --env-file deploy/env/server.env \ + -f compose.yaml -f deploy/compose.server.yaml \ + -f deploy/compose.session-server.yaml.example build core +docker compose --env-file deploy/env/server.env \ + -f compose.yaml -f deploy/compose.server.yaml \ + -f deploy/compose.session-server.yaml.example up -d catalog-db +docker compose --env-file deploy/env/server.env \ + -f compose.yaml -f deploy/compose.server.yaml \ + -f deploy/compose.session-server.yaml.example run --rm catalog-migrate docker compose --env-file deploy/env/server.env \ -f compose.yaml -f deploy/compose.server.yaml \ -f deploy/compose.session-server.yaml.example up --build -d @@ -101,10 +113,12 @@ schema, Evidence, and Memory records share that collection and stay separated by `kind`. -Connector `ssh_tunnel` bindings are diagnostic-only in this release: their bounded probe always -cleans up the loopback forward and returns `workspace_not_activatable`; session creation is rejected -before persistence. Git registry access over SSH is unaffected. Use direct or REST connector -transport for runtime sessions. +For NL→SQL runtime sessions, connector `ssh_tunnel` bindings remain diagnostic-only: their bounded +probe cleans up the loopback forward and returns `workspace_not_activatable`; session creation is +rejected before persistence. Database management is a separate boundary and supports a strict +OpenSSH tunnel for **Test connection** and **Sync tables**, using a private key, optional passphrase, +mandatory `known_hosts`, and optional PostgreSQL TLS CA/server name. Git registry access over SSH is +unaffected. Use direct or REST connector transport for runtime sessions. `docker-compose.dev.yml` is deliberately local: both published ports bind to `127.0.0.1`, `THT_SESSION_STORAGE=local`, and `THT_HOME=/data/local-home`. Do not set diff --git a/backend/package-lock.json b/backend/package-lock.json index 9478eec4..1d8cd11f 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -11,18 +11,27 @@ "@fastify/rate-limit": "11.2.0", "@types/pg": "^8.20.3", "fastify": "^5.0.0", + "kysely": "^0.29.5", "openid-client": "6.8.5", "pg": "^8.22.0", "yaml": "^2.9.0", "zod": "^4.4.3" }, "devDependencies": { + "@testcontainers/postgresql": "^12.1.0", "@types/node": "24.13.3", "tsx": "^4.19.0", "typescript": "^5.6.0", "vitest": "^2.1.0" } }, + "node_modules/@balena/dockerignore": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@balena/dockerignore/-/dockerignore-1.0.2.tgz", + "integrity": "sha512-wMue2Sy4GAVTk6Ic4tJVcnfdau+gx2EnG7S+uAEe+TWJFqE4YoWN4/H8MSLj4eYJKxGg26lZwboEniNiNwZQ6Q==", + "dev": true, + "license": "Apache-2.0" + }, "node_modules/@esbuild/aix-ppc64": { "version": "0.28.1", "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.28.1.tgz", @@ -683,6 +692,76 @@ ], "license": "MIT" }, + "node_modules/@grpc/grpc-js": { + "version": "1.14.4", + "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.14.4.tgz", + "integrity": "sha512-k9Dj3DV/itK9D06Y8f190Qgop7/Ui+D0njFV3LHMPwPT75DpXLQohE9Wmz0QElrJnzsjB7KPWiKJbOl7IPDArQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@grpc/proto-loader": "^0.8.0", + "@js-sdsl/ordered-map": "^4.4.2" + }, + "engines": { + "node": ">=12.10.0" + } + }, + "node_modules/@grpc/grpc-js/node_modules/@grpc/proto-loader": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.8.1.tgz", + "integrity": "sha512-wtF6h+DY6M3YaDBPAmvuuA6jV8Sif9MjtOI5euKFWRgCDl5PeDpPsHR9u2l6St5ceY8AZgoNDww5+HvEsXFsGg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "lodash.camelcase": "^4.3.0", + "long": "^5.0.0", + "protobufjs": "^7.5.5", + "yargs": "^17.7.2" + }, + "bin": { + "proto-loader-gen-types": "build/bin/proto-loader-gen-types.js" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/@grpc/proto-loader": { + "version": "0.7.15", + "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.7.15.tgz", + "integrity": "sha512-tMXdRCfYVixjuFK+Hk0Q1s38gV9zDiDJfWL3h1rv4Qc39oILCu1TRTDt7+fGUI8K4G1Fj125Hx/ru3azECWTyQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "lodash.camelcase": "^4.3.0", + "long": "^5.0.0", + "protobufjs": "^7.2.5", + "yargs": "^17.7.2" + }, + "bin": { + "proto-loader-gen-types": "build/bin/proto-loader-gen-types.js" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/@isaacs/cliui": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", + "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", + "dev": true, + "license": "ISC", + "dependencies": { + "string-width": "^5.1.2", + "string-width-cjs": "npm:string-width@^4.2.0", + "strip-ansi": "^7.0.1", + "strip-ansi-cjs": "npm:strip-ansi@^6.0.1", + "wrap-ansi": "^8.1.0", + "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, "node_modules/@jridgewell/sourcemap-codec": { "version": "1.5.5", "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", @@ -690,6 +769,27 @@ "dev": true, "license": "MIT" }, + "node_modules/@js-sdsl/ordered-map": { + "version": "4.4.2", + "resolved": "https://registry.npmjs.org/@js-sdsl/ordered-map/-/ordered-map-4.4.2.tgz", + "integrity": "sha512-iUKgm52T8HOE/makSxjqoWhe95ZJA1/G1sYsGev2JDKUSS14KAgg1LHb+Ba+IPow0xflbnSkOsZcO08C7w1gYw==", + "dev": true, + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/js-sdsl" + } + }, + "node_modules/@kwsites/file-exists": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@kwsites/file-exists/-/file-exists-1.1.1.tgz", + "integrity": "sha512-m9/5YGR18lIwxSFDwfE3oA7bWuq9kdau6ugN4H2rJeyhFQZcG9AgSHkQtSD15a8WvTgfz9aikZMrKPHvbpqFiw==", + "dev": true, + "license": "MIT", + "dependencies": { + "debug": "^4.1.1" + } + }, "node_modules/@lukeed/ms": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/@lukeed/ms/-/ms-2.0.2.tgz", @@ -705,6 +805,83 @@ "integrity": "sha512-k2ENnmBugE/rzQfEcdWHcCY+/FM3VLzH9cYEsbdsoqrvzAKRhUZeRNhAZvB8OitQJ1TBed3yqWtdjzS6wJKBwg==", "license": "MIT" }, + "node_modules/@pkgjs/parseargs": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", + "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==", + "dev": true, + "license": "MIT", + "optional": true, + "engines": { + "node": ">=14" + } + }, + "node_modules/@protobufjs/aspromise": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", + "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==", + "dev": true, + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/base64": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", + "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==", + "dev": true, + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/codegen": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.5.tgz", + "integrity": "sha512-zgXFLzW3Ap33e6d0Wlj4MGIm6Ce8O89n/apUaGNB/jx+hw+ruWEp7EwGUshdLKVRCxZW12fp9r40E1mQrf/34g==", + "dev": true, + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/eventemitter": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.1.tgz", + "integrity": "sha512-vW1GmwMZNnL+gMRaovlh9yZX74kc+TTU3FObkkurpMaRtBfLP3ldjS9KQWlwZgraRE0+dheEEoAxdzcJQ8eXZg==", + "dev": true, + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/fetch": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.1.tgz", + "integrity": "sha512-GpptLrs57adMSuHi3VNj0mAF8dwh36LMaYF6XyJ6JMWlVsc+t42tm1HSEDmOs3A8fC9yyeisgLhsTVQokOZ0zw==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.1" + } + }, + "node_modules/@protobufjs/float": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", + "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==", + "dev": true, + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/path": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", + "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==", + "dev": true, + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/pool": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", + "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==", + "dev": true, + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/utf8": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.2.tgz", + "integrity": "sha512-b1UQwcEZ4yCnMCD8DAL1VlbvBJE9/IX4FTIp7BG1xYpf29SLazLSrqUkj4w7Y5y7cCVP6E5tcqqcI0xemPkHug==", + "dev": true, + "license": "BSD-3-Clause" + }, "node_modules/@rollup/rollup-android-arm-eabi": { "version": "4.62.2", "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.62.2.tgz", @@ -1055,6 +1232,39 @@ "win32" ] }, + "node_modules/@testcontainers/postgresql": { + "version": "12.1.0", + "resolved": "https://registry.npmjs.org/@testcontainers/postgresql/-/postgresql-12.1.0.tgz", + "integrity": "sha512-Pjf2VSVNirEPfz36nidyrVAnZvc2YhajOznY4VgyEsvfTd5qiMNOuPq96drREvxAUtXl5SFLX7vXj7sSq4aTcA==", + "dev": true, + "license": "MIT", + "dependencies": { + "testcontainers": "^12.1.0" + } + }, + "node_modules/@types/docker-modem": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/@types/docker-modem/-/docker-modem-3.0.6.tgz", + "integrity": "sha512-yKpAGEuKRSS8wwx0joknWxsmLha78wNMe9R2S3UNsVOkZded8UqOrV8KoeDXoXsjndxwyF3eIhyClGbO1SEhEg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*", + "@types/ssh2": "*" + } + }, + "node_modules/@types/dockerode": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@types/dockerode/-/dockerode-4.0.1.tgz", + "integrity": "sha512-cmUpB+dPN955PxBEuXE3f6lKO1hHiIGYJA46IVF3BJpNsZGvtBDcRnlrHYHtOH/B6vtDOyl2kZ2ShAu3mgc27Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/docker-modem": "*", + "@types/node": "*", + "@types/ssh2": "*" + } + }, "node_modules/@types/estree": { "version": "1.0.9", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.9.tgz", @@ -1082,6 +1292,43 @@ "pg-types": "^2.2.0" } }, + "node_modules/@types/ssh2": { + "version": "1.15.5", + "resolved": "https://registry.npmjs.org/@types/ssh2/-/ssh2-1.15.5.tgz", + "integrity": "sha512-N1ASjp/nXH3ovBHddRJpli4ozpk6UdDYIX4RJWFa9L1YKnzdhTlVmiGHm4DZnj/jLbqZpes4aeR30EFGQtvhQQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "^18.11.18" + } + }, + "node_modules/@types/ssh2-streams": { + "version": "0.1.13", + "resolved": "https://registry.npmjs.org/@types/ssh2-streams/-/ssh2-streams-0.1.13.tgz", + "integrity": "sha512-faHyY3brO9oLEA0QlcO8N2wT7R0+1sHWZvQ+y3rMLwdY1ZyS1z0W3t65j9PqT4HmQ6ALzNe7RZlNuCNE0wBSWA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/ssh2/node_modules/@types/node": { + "version": "18.19.130", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.130.tgz", + "integrity": "sha512-GRaXQx6jGfL8sKfaIDD6OupbIHBr9jv7Jnaml9tB7l4v068PAOXqfcujMMo5PhbIs6ggR1XODELqahT2R8v0fg==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~5.26.4" + } + }, + "node_modules/@types/ssh2/node_modules/undici-types": { + "version": "5.26.5", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", + "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", + "dev": true, + "license": "MIT" + }, "node_modules/@vitest/expect": { "version": "2.1.9", "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-2.1.9.tgz", @@ -1195,6 +1442,19 @@ "url": "https://opencollective.com/vitest" } }, + "node_modules/abort-controller": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", + "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", + "dev": true, + "license": "MIT", + "dependencies": { + "event-target-shim": "^5.0.0" + }, + "engines": { + "node": ">=6.5" + } + }, "node_modules/abstract-logging": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/abstract-logging/-/abstract-logging-2.0.1.tgz", @@ -1234,6 +1494,80 @@ } } }, + "node_modules/ansi-regex": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.3.0.tgz", + "integrity": "sha512-WpDfL7NO6j7tH88IDBNVdUJxDh9nmCteAVW9dsep846XdwF4naCBK+/tGLX3KJgcpgMRXCFlTM2hKGoK9FsdrQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-regex?sponsor=1" + } + }, + "node_modules/ansi-styles": { + "version": "6.2.3", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.3.tgz", + "integrity": "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/archiver": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/archiver/-/archiver-7.0.1.tgz", + "integrity": "sha512-ZcbTaIqJOfCc03QwD468Unz/5Ir8ATtvAHsK+FdXbDIbGfihqh9mrvdcYunQzqn4HrvWWaFyaxJhGZagaJJpPQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "archiver-utils": "^5.0.2", + "async": "^3.2.4", + "buffer-crc32": "^1.0.0", + "readable-stream": "^4.0.0", + "readdir-glob": "^1.1.2", + "tar-stream": "^3.0.0", + "zip-stream": "^6.0.1" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/archiver-utils": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/archiver-utils/-/archiver-utils-5.0.2.tgz", + "integrity": "sha512-wuLJMmIBQYCsGZgYLTy5FIB2pF6Lfb6cXMSF8Qywwk3t20zWnAi7zLcQFdKQmIB8wyZpY5ER38x08GbwtR2cLA==", + "dev": true, + "license": "MIT", + "dependencies": { + "glob": "^10.0.0", + "graceful-fs": "^4.2.0", + "is-stream": "^2.0.1", + "lazystream": "^1.0.0", + "lodash": "^4.17.15", + "normalize-path": "^3.0.0", + "readable-stream": "^4.0.0" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/asn1": { + "version": "0.2.6", + "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.6.tgz", + "integrity": "sha512-ix/FxPn0MDjeyJ7i/yoHGFt/EX6LyNbxSEhPPXODPL+KB0VPk86UYfL0lMdy+KCnv+fmvIzySwaK5COwqVbWTQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "safer-buffer": "~2.1.0" + } + }, "node_modules/assertion-error": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-2.0.1.tgz", @@ -1244,6 +1578,20 @@ "node": ">=12" } }, + "node_modules/async": { + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/async/-/async-3.2.6.tgz", + "integrity": "sha512-htCUDlxyyCLMgaM3xXg0C0LW2xqfuQ6p05pCEIsXuyQ+a1koYKTuBMzRNwmybfLgvJDMd0r1LTn4+E0Ti6C2AA==", + "dev": true, + "license": "MIT" + }, + "node_modules/async-lock": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/async-lock/-/async-lock-1.4.1.tgz", + "integrity": "sha512-Az2ZTpuytrtqENulXwO3GGv1Bztugx6TT37NIo7imr/Qo0gsYiGtSdBa2B6fsXhTpVZDNfu1Qn3pk531e3q+nQ==", + "dev": true, + "license": "MIT" + }, "node_modules/atomic-sleep": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/atomic-sleep/-/atomic-sleep-1.0.0.tgz", @@ -1273,6 +1621,261 @@ "fastq": "^1.17.1" } }, + "node_modules/b4a": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/b4a/-/b4a-1.8.1.tgz", + "integrity": "sha512-aiqre1Nr0B/6DgE2N5vwTc+2/oQZ4Wh1t4NznYY4E00y8LCt6NqdRv81so00oo27D8MVKTpUa/MwUUtBLXCoDw==", + "dev": true, + "license": "Apache-2.0", + "peerDependencies": { + "react-native-b4a": "*" + }, + "peerDependenciesMeta": { + "react-native-b4a": { + "optional": true + } + } + }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "dev": true, + "license": "MIT" + }, + "node_modules/bare-events": { + "version": "2.9.2", + "resolved": "https://registry.npmjs.org/bare-events/-/bare-events-2.9.2.tgz", + "integrity": "sha512-AIPKioV7/Y/8KfZ3AAhjPJxLLbY49S64Ym5DakZlUg75qQiTgUq9hEJoEwa4eUezPUlXRy/i5NpsKvo9jgKmoA==", + "dev": true, + "license": "Apache-2.0", + "peerDependencies": { + "bare-abort-controller": "*" + }, + "peerDependenciesMeta": { + "bare-abort-controller": { + "optional": true + } + } + }, + "node_modules/bare-fs": { + "version": "4.8.1", + "resolved": "https://registry.npmjs.org/bare-fs/-/bare-fs-4.8.1.tgz", + "integrity": "sha512-N1nnXdHZAOSstz0XiHikGS4HGMH4CnSwhqWdGQQMqqdvp4Jybm9sE3R1WVnpWVd4SFkc8ryPDBLViNLwiEqECg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "bare-events": "^2.5.4", + "bare-path": "^3.0.0", + "bare-stream": "^2.6.4", + "bare-url": "^2.2.2", + "fast-fifo": "^1.3.2" + }, + "engines": { + "bare": ">=1.28.0" + }, + "peerDependencies": { + "bare-buffer": "*" + }, + "peerDependenciesMeta": { + "bare-buffer": { + "optional": true + } + } + }, + "node_modules/bare-path": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/bare-path/-/bare-path-3.1.1.tgz", + "integrity": "sha512-JprUlveX3QjApC1cTpsUOiscADftCGVWkzitbHsRqv84hzYwYHw2mbluddsq5TvI8mH/8Ov1f4BiMAdcB0oYnQ==", + "dev": true, + "license": "Apache-2.0" + }, + "node_modules/bare-stream": { + "version": "2.13.4", + "resolved": "https://registry.npmjs.org/bare-stream/-/bare-stream-2.13.4.tgz", + "integrity": "sha512-PcrQ8lVLbiJscNm1Kez+Yp4Gy4AHGcN1lzwjvf5NybWen7VvEgUfyfnXYJ2zNqWnzOfCb1Abq6lH8ti0syQszA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "b4a": "^1.8.1", + "streamx": "^2.25.0", + "teex": "^1.0.1" + }, + "peerDependencies": { + "bare-abort-controller": "*", + "bare-buffer": "*", + "bare-events": "*" + }, + "peerDependenciesMeta": { + "bare-abort-controller": { + "optional": true + }, + "bare-buffer": { + "optional": true + }, + "bare-events": { + "optional": true + } + } + }, + "node_modules/bare-url": { + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/bare-url/-/bare-url-2.5.2.tgz", + "integrity": "sha512-L13PCJzKG8RGvx8V1/DdMi12ERhC3tprr7/8a94BxpmnRsFqxh5XZNdhtMxu5HPkRshYOOWRGY8lDP7ZhpG9Cg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "bare-path": "^3.0.0" + } + }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/bcrypt-pbkdf": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", + "integrity": "sha512-qeFIXtP4MSoi6NLqO12WfqARWWuCKi2Rn/9hJLEmtB5yTNr9DqFWkJRCf2qShWzPeAMRnOgCrq0sg/KLv5ES9w==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "tweetnacl": "^0.14.3" + } + }, + "node_modules/bl": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", + "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", + "dev": true, + "license": "MIT", + "dependencies": { + "buffer": "^5.5.0", + "inherits": "^2.0.4", + "readable-stream": "^3.4.0" + } + }, + "node_modules/bl/node_modules/buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, + "node_modules/bl/node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "dev": true, + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/brace-expansion": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.4.tgz", + "integrity": "sha512-hGfVzPxthbf3+2yjg/RBs60cB0FhqBS/zvdV/4wn4/BmN0bNMMHPc4V/BbFieqf1TKAGGAHnY4eSjajCl0f2Xg==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/buffer": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", + "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.2.1" + } + }, + "node_modules/buffer-crc32": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-1.0.0.tgz", + "integrity": "sha512-Db1SbgBS/fg/392AblrMJk97KggmvYhr4pB5ZIMTWtaivCPMWLkmb7m21cJvpvgK+J3nsU2CmmixNBZx4vFj/w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/buildcheck": { + "version": "0.0.7", + "resolved": "https://registry.npmjs.org/buildcheck/-/buildcheck-0.0.7.tgz", + "integrity": "sha512-lHblz4ahamxpTmnsk+MNTRWsjYKv965MwOrSJyeD588rR3Jcu7swE+0wN5F+PbL5cjgu/9ObkhfzEPuofEMwLA==", + "dev": true, + "optional": true, + "engines": { + "node": ">=10.0.0" + } + }, + "node_modules/byline": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/byline/-/byline-5.0.0.tgz", + "integrity": "sha512-s6webAy+R4SR8XVuJWt2V2rGvhnrhxN+9S15GNuTK3wKPOXFF6RNc+8ug2XhH+2s4f+uudG4kUVYmYOQWL2g0Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/cac": { "version": "6.7.14", "resolved": "https://registry.npmjs.org/cac/-/cac-6.7.14.tgz", @@ -1310,6 +1913,144 @@ "node": ">= 16" } }, + "node_modules/chownr": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==", + "dev": true, + "license": "ISC" + }, + "node_modules/cliui": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", + "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", + "dev": true, + "license": "ISC", + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.1", + "wrap-ansi": "^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/cliui/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/cliui/node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/cliui/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true, + "license": "MIT" + }, + "node_modules/cliui/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/cliui/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/cliui/node_modules/wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true, + "license": "MIT" + }, + "node_modules/compress-commons": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/compress-commons/-/compress-commons-6.0.2.tgz", + "integrity": "sha512-6FqVXeETqWPoGcfzrXb37E50NP0LXT8kAMu5ooZayhWWdgEY4lBEEcbQNXtkuKQsGduxiIcI4gOTsxTmuq/bSg==", + "dev": true, + "license": "MIT", + "dependencies": { + "crc-32": "^1.2.0", + "crc32-stream": "^6.0.0", + "is-stream": "^2.0.1", + "normalize-path": "^3.0.0", + "readable-stream": "^4.0.0" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/cookie": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/cookie/-/cookie-1.1.1.tgz", @@ -1323,6 +2064,70 @@ "url": "https://opencollective.com/express" } }, + "node_modules/core-util-is": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", + "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/cpu-features": { + "version": "0.0.10", + "resolved": "https://registry.npmjs.org/cpu-features/-/cpu-features-0.0.10.tgz", + "integrity": "sha512-9IkYqtX3YHPCzoVg1Py+o9057a3i0fp7S530UWokCSaFVTc7CwXPRiOjRjBQQ18ZCNafx78YfnG+HALxtVmOGA==", + "dev": true, + "hasInstallScript": true, + "optional": true, + "dependencies": { + "buildcheck": "~0.0.6", + "nan": "^2.19.0" + }, + "engines": { + "node": ">=10.0.0" + } + }, + "node_modules/crc-32": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/crc-32/-/crc-32-1.2.2.tgz", + "integrity": "sha512-ROmzCKrTnOwybPcJApAA6WBWij23HVfGVNKqqrZpuyZOHqK2CwHSvpGuyt/UNNvaIjEd8X5IFGp4Mh+Ie1IHJQ==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "crc32": "bin/crc32.njs" + }, + "engines": { + "node": ">=0.8" + } + }, + "node_modules/crc32-stream": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/crc32-stream/-/crc32-stream-6.0.0.tgz", + "integrity": "sha512-piICUB6ei4IlTv1+653yq5+KoqfBYmj9bw6LqXoOneTMDXk5nM1qt12mFW1caG3LlJXEKW1Bp0WggEmIfQB34g==", + "dev": true, + "license": "MIT", + "dependencies": { + "crc-32": "^1.2.0", + "readable-stream": "^4.0.0" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "dev": true, + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, "node_modules/debug": { "version": "4.4.3", "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", @@ -1360,6 +2165,137 @@ "node": ">=6" } }, + "node_modules/docker-compose": { + "version": "1.4.2", + "resolved": "https://registry.npmjs.org/docker-compose/-/docker-compose-1.4.2.tgz", + "integrity": "sha512-rPHigTKGaEHpkUmfd69QgaOp+Os5vGJwG/Ry8lcr8W/382AmI+z/D7qoa9BybKIkqNppaIbs8RYeHSevdQjWww==", + "dev": true, + "license": "MIT", + "dependencies": { + "yaml": "^2.2.2" + }, + "engines": { + "node": ">= 6.0.0" + } + }, + "node_modules/docker-modem": { + "version": "5.0.7", + "resolved": "https://registry.npmjs.org/docker-modem/-/docker-modem-5.0.7.tgz", + "integrity": "sha512-XJgGhoR/CLpqshm4d3L7rzH6t8NgDFUIIpztYlLHIApeJjMZKYJMz2zxPsYxnejq5h3ELYSw/RBsi3t5h7gNTA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "debug": "^4.1.1", + "readable-stream": "^3.5.0", + "split-ca": "^1.0.1", + "ssh2": "^1.15.0" + }, + "engines": { + "node": ">= 8.0" + } + }, + "node_modules/docker-modem/node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "dev": true, + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/dockerode": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/dockerode/-/dockerode-5.0.1.tgz", + "integrity": "sha512-avsq/xk4YPIrn0CgleX5bjT9Y8IT1p9PxrNQ++RBQ2WEyFfHCTDsT9kmyxz+H/axnjAwg8wJWEIuPGOUuNupiA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@balena/dockerignore": "^1.0.2", + "@grpc/grpc-js": "^1.11.1", + "@grpc/proto-loader": "^0.7.13", + "docker-modem": "^5.0.7", + "protobufjs": "^7.3.2", + "tar-fs": "^2.1.4" + }, + "engines": { + "node": ">= 14.17" + } + }, + "node_modules/dockerode/node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "dev": true, + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/dockerode/node_modules/tar-fs": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.5.tgz", + "integrity": "sha512-OboTd8mmMhZDNPV+UjQcK9yKAatXu2aJ+r1w4im1Otd4M4fl2hwvdoXUxIYHFTHWK/3y3FarBP70v3vwmGlOxw==", + "dev": true, + "license": "MIT", + "dependencies": { + "chownr": "^1.1.1", + "mkdirp-classic": "^0.5.2", + "pump": "^3.0.0", + "tar-stream": "^2.1.4" + } + }, + "node_modules/dockerode/node_modules/tar-stream": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", + "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "bl": "^4.0.3", + "end-of-stream": "^1.4.1", + "fs-constants": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/eastasianwidth": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", + "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==", + "dev": true, + "license": "MIT" + }, + "node_modules/emoji-regex": { + "version": "9.2.2", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", + "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==", + "dev": true, + "license": "MIT" + }, + "node_modules/end-of-stream": { + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz", + "integrity": "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==", + "dev": true, + "license": "MIT", + "dependencies": { + "once": "^1.4.0" + } + }, "node_modules/es-module-lexer": { "version": "1.7.0", "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-1.7.0.tgz", @@ -1409,6 +2345,16 @@ "@esbuild/win32-x64": "0.28.1" } }, + "node_modules/escalade": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", + "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, "node_modules/estree-walker": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-3.0.3.tgz", @@ -1419,6 +2365,36 @@ "@types/estree": "^1.0.0" } }, + "node_modules/event-target-shim": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", + "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/events": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", + "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.8.x" + } + }, + "node_modules/events-universal": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/events-universal/-/events-universal-1.0.1.tgz", + "integrity": "sha512-LUd5euvbMLpwOF8m6ivPCbhQeSiYVNb8Vs0fQ8QjXo0JTkEHpz8pxdQf0gStltaPpw0Cca8b39KxvK9cfKRiAw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "bare-events": "^2.7.0" + } + }, "node_modules/expect-type": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/expect-type/-/expect-type-1.4.0.tgz", @@ -1441,6 +2417,13 @@ "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", "license": "MIT" }, + "node_modules/fast-fifo": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/fast-fifo/-/fast-fifo-1.3.2.tgz", + "integrity": "sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==", + "dev": true, + "license": "MIT" + }, "node_modules/fast-json-stringify": { "version": "6.4.0", "resolved": "https://registry.npmjs.org/fast-json-stringify/-/fast-json-stringify-6.4.0.tgz", @@ -1562,6 +2545,30 @@ "node": ">=20" } }, + "node_modules/foreground-child": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.3.1.tgz", + "integrity": "sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw==", + "dev": true, + "license": "ISC", + "dependencies": { + "cross-spawn": "^7.0.6", + "signal-exit": "^4.0.1" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/fs-constants": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", + "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==", + "dev": true, + "license": "MIT" + }, "node_modules/fsevents": { "version": "2.3.3", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", @@ -1577,6 +2584,86 @@ "node": "^8.16.0 || ^10.6.0 || >=11.0.0" } }, + "node_modules/get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "dev": true, + "license": "ISC", + "engines": { + "node": "6.* || 8.* || >= 10.*" + } + }, + "node_modules/get-port": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/get-port/-/get-port-5.1.1.tgz", + "integrity": "sha512-g/Q1aTSDOxFpchXC4i8ZWvxA1lnPqx/JHqcpIw0/LX9T8x/GBbi6YnlN5nhaKIFkT8oFsscUKgDJYxfwfS6QsQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/glob": { + "version": "10.5.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-10.5.0.tgz", + "integrity": "sha512-DfXN8DfhJ7NH3Oe7cFmu3NCu1wKbkReJ8TorzSAFbSKrlNaQSKfIzqYqVY8zlbs2NLBbWpRiU52GX2PbaBVNkg==", + "deprecated": "Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me", + "dev": true, + "license": "ISC", + "dependencies": { + "foreground-child": "^3.1.0", + "jackspeak": "^3.1.2", + "minimatch": "^9.0.4", + "minipass": "^7.1.2", + "package-json-from-dist": "^1.0.0", + "path-scurry": "^1.11.1" + }, + "bin": { + "glob": "dist/esm/bin.mjs" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/graceful-fs": { + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "dev": true, + "license": "ISC" + }, "node_modules/ip-address": { "version": "10.5.0", "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.5.0.tgz", @@ -1595,6 +2682,59 @@ "node": ">= 10" } }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/is-stream": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", + "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "dev": true, + "license": "ISC" + }, + "node_modules/jackspeak": { + "version": "3.4.3", + "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-3.4.3.tgz", + "integrity": "sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "@isaacs/cliui": "^8.0.2" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + }, + "optionalDependencies": { + "@pkgjs/parseargs": "^0.11.0" + } + }, "node_modules/jose": { "version": "6.2.9", "resolved": "https://registry.npmjs.org/jose/-/jose-6.2.9.tgz", @@ -1629,6 +2769,61 @@ "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", "license": "MIT" }, + "node_modules/kysely": { + "version": "0.29.5", + "resolved": "https://registry.npmjs.org/kysely/-/kysely-0.29.5.tgz", + "integrity": "sha512-ooa+eSbBNPTo3MycPEuW5jdrxQdQwdtB3LC3h43FiXQbIry5tR0C5lDG7eealK0E4D7XjrnOP5DIUg/LyjRMYQ==", + "license": "MIT", + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/lazystream": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/lazystream/-/lazystream-1.0.1.tgz", + "integrity": "sha512-b94GiNHQNy6JNTrt5w6zNyffMrNkXZb3KTkCZJb2V1xaEGCk093vkZ2jk3tpaeP33/OiXC+WvK9AxUebnf5nbw==", + "dev": true, + "license": "MIT", + "dependencies": { + "readable-stream": "^2.0.5" + }, + "engines": { + "node": ">= 0.6.3" + } + }, + "node_modules/lazystream/node_modules/readable-stream": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "dev": true, + "license": "MIT", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/lazystream/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "dev": true, + "license": "MIT" + }, + "node_modules/lazystream/node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "dev": true, + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, "node_modules/light-my-request": { "version": "6.6.0", "resolved": "https://registry.npmjs.org/light-my-request/-/light-my-request-6.6.0.tgz", @@ -1666,6 +2861,27 @@ ], "license": "MIT" }, + "node_modules/lodash": { + "version": "4.18.1", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.18.1.tgz", + "integrity": "sha512-dMInicTPVE8d1e5otfwmmjlxkZoUpiVLwyeTdUsi/Caj/gfzzblBcCE5sRHV/AsjuCmxWrte2TNGSYuCeCq+0Q==", + "dev": true, + "license": "MIT" + }, + "node_modules/lodash.camelcase": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz", + "integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==", + "dev": true, + "license": "MIT" + }, + "node_modules/long": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", + "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==", + "dev": true, + "license": "Apache-2.0" + }, "node_modules/loupe": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/loupe/-/loupe-3.2.1.tgz", @@ -1673,6 +2889,13 @@ "dev": true, "license": "MIT" }, + "node_modules/lru-cache": { + "version": "10.4.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.4.3.tgz", + "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==", + "dev": true, + "license": "ISC" + }, "node_modules/magic-string": { "version": "0.30.21", "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.21.tgz", @@ -1683,6 +2906,55 @@ "@jridgewell/sourcemap-codec": "^1.5.5" } }, + "node_modules/minimatch": { + "version": "9.0.9", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.9.tgz", + "integrity": "sha512-OBwBN9AL4dqmETlpS2zasx+vTeWclWzkblfZk7KTA5j3jeOONz/tRCnZomUyvNg83wL5Zv9Ss6HMJXAgL8R2Yg==", + "dev": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^2.0.2" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/minipass": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.3.tgz", + "integrity": "sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==", + "dev": true, + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, + "node_modules/mkdirp": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-3.0.1.tgz", + "integrity": "sha512-+NsyUUAZDmo6YVHzL/stxSu3t9YS1iljliy3BSDrXJ/dkn1KYdmtZODGGjLcc9XLgVVpH4KshHB8XmZgMhaBXg==", + "dev": true, + "license": "MIT", + "bin": { + "mkdirp": "dist/cjs/src/bin.js" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/mkdirp-classic": { + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", + "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==", + "dev": true, + "license": "MIT" + }, "node_modules/ms": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", @@ -1690,6 +2962,14 @@ "dev": true, "license": "MIT" }, + "node_modules/nan": { + "version": "2.28.0", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.28.0.tgz", + "integrity": "sha512-fTsDz99OTq2sVePhGdp4qQhggZFtKr64ZNVyVajRKtMOkJxYekplBh577PiJB12v/D3s2E5cGtOI45LWp6rnLQ==", + "dev": true, + "license": "MIT", + "optional": true + }, "node_modules/nanoid": { "version": "3.3.15", "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.15.tgz", @@ -1709,6 +2989,16 @@ "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" } }, + "node_modules/normalize-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", + "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/oauth4webapi": { "version": "3.8.7", "resolved": "https://registry.npmjs.org/oauth4webapi/-/oauth4webapi-3.8.7.tgz", @@ -1727,6 +3017,16 @@ "node": ">=14.0.0" } }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "dev": true, + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, "node_modules/openid-client": { "version": "6.8.5", "resolved": "https://registry.npmjs.org/openid-client/-/openid-client-6.8.5.tgz", @@ -1740,6 +3040,40 @@ "url": "https://github.com/sponsors/panva" } }, + "node_modules/package-json-from-dist": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/package-json-from-dist/-/package-json-from-dist-1.0.1.tgz", + "integrity": "sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==", + "dev": true, + "license": "BlueOak-1.0.0" + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-scurry": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.11.1.tgz", + "integrity": "sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA==", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "lru-cache": "^10.2.0", + "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0" + }, + "engines": { + "node": ">=16 || 14 >=14.18" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/pathe": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/pathe/-/pathe-1.1.2.tgz", @@ -1958,6 +3292,23 @@ "node": ">=0.10.0" } }, + "node_modules/process": { + "version": "0.11.10", + "resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz", + "integrity": "sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6.0" + } + }, + "node_modules/process-nextick-args": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", + "dev": true, + "license": "MIT" + }, "node_modules/process-warning": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/process-warning/-/process-warning-5.0.0.tgz", @@ -1974,12 +3325,124 @@ ], "license": "MIT" }, + "node_modules/proper-lockfile": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/proper-lockfile/-/proper-lockfile-4.1.2.tgz", + "integrity": "sha512-TjNPblN4BwAWMXU8s9AEz4JmQxnD1NNL7bNOY/AKUzyamc379FWASUhc/K1pL2noVb+XmZKLL68cjzLsiOAMaA==", + "dev": true, + "license": "MIT", + "dependencies": { + "graceful-fs": "^4.2.4", + "retry": "^0.12.0", + "signal-exit": "^3.0.2" + } + }, + "node_modules/proper-lockfile/node_modules/signal-exit": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", + "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/properties-reader": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/properties-reader/-/properties-reader-3.0.1.tgz", + "integrity": "sha512-WPn+h9RGEExOKdu4bsF4HksG/uzd3cFq3MFtq8PsFeExPse5Ha/VOjQNyHhjboBFwGXGev6muJYTSPAOkROq2g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@kwsites/file-exists": "^1.1.1", + "mkdirp": "^3.0.1" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "github", + "url": "https://github.com/steveukx/properties?sponsor=1" + } + }, + "node_modules/protobufjs": { + "version": "7.6.5", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.6.5.tgz", + "integrity": "sha512-/FPD0nUc9jH6rfFjji9IBqOz4pcSE3CsT1m7Ep6Mdb0LxSUMj8hgl6GomOvZzpNpAqqGaXA0P3VSrZLFzIhQrw==", + "dev": true, + "hasInstallScript": true, + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.5", + "@protobufjs/eventemitter": "^1.1.1", + "@protobufjs/fetch": "^1.1.1", + "@protobufjs/float": "^1.0.2", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.1", + "@types/node": ">=13.7.0", + "long": "^5.3.2" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/pump": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.4.tgz", + "integrity": "sha512-VS7sjc6KR7e1ukRFhQSY5LM2uBWAUPiOPa/A3mkKmiMwSmRFUITt0xuj+/lesgnCv+dPIEYlkzrcyXgquIHMcA==", + "dev": true, + "license": "MIT", + "dependencies": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, "node_modules/quick-format-unescaped": { "version": "4.0.4", "resolved": "https://registry.npmjs.org/quick-format-unescaped/-/quick-format-unescaped-4.0.4.tgz", "integrity": "sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg==", "license": "MIT" }, + "node_modules/readable-stream": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.7.0.tgz", + "integrity": "sha512-oIGGmcpTLwPga8Bn6/Z75SVaH1z5dUut2ibSyAMVhmUggWpmDn2dapB0n7f8nwaSiRtepAsfJyfXIO5DCVAODg==", + "dev": true, + "license": "MIT", + "dependencies": { + "abort-controller": "^3.0.0", + "buffer": "^6.0.3", + "events": "^3.3.0", + "process": "^0.11.10", + "string_decoder": "^1.3.0" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + } + }, + "node_modules/readdir-glob": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/readdir-glob/-/readdir-glob-1.1.3.tgz", + "integrity": "sha512-v05I2k7xN8zXvPD9N+z/uhXPaj0sUFCe2rcWZIpBsqxfP7xXFQ0tipAd/wjj1YxWyWtUS5IDJpOG82JKt2EAVA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "minimatch": "^5.1.0" + } + }, + "node_modules/readdir-glob/node_modules/minimatch": { + "version": "5.1.9", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.9.tgz", + "integrity": "sha512-7o1wEA2RyMP7Iu7GNba9vc0RWWGACJOCZBJX2GJWip0ikV+wcOsgVuY9uE8CPiyQhkGFSlhuSkZPavN7u1c2Fw==", + "dev": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/real-require": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/real-require/-/real-require-0.2.0.tgz", @@ -1989,6 +3452,16 @@ "node": ">= 12.13.0" } }, + "node_modules/require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/require-from-string": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", @@ -2007,6 +3480,16 @@ "node": ">=10" } }, + "node_modules/retry": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/retry/-/retry-0.12.0.tgz", + "integrity": "sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, "node_modules/reusify": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.1.0.tgz", @@ -2068,6 +3551,27 @@ "fsevents": "~2.3.2" } }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, "node_modules/safe-regex2": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/safe-regex2/-/safe-regex2-5.1.1.tgz", @@ -2099,6 +3603,13 @@ "node": ">=10" } }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "dev": true, + "license": "MIT" + }, "node_modules/secure-json-parse": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/secure-json-parse/-/secure-json-parse-4.1.0.tgz", @@ -2133,6 +3644,29 @@ "integrity": "sha512-oeM1lpU/UvhTxw+g3cIfxXHyJRc/uidd3yK1P242gzHds0udQBYzs3y8j4gCCW+ZJ7ad0yctld8RYO+bdurlvw==", "license": "MIT" }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dev": true, + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, "node_modules/siginfo": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/siginfo/-/siginfo-2.0.0.tgz", @@ -2140,6 +3674,19 @@ "dev": true, "license": "ISC" }, + "node_modules/signal-exit": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/sonic-boom": { "version": "4.2.1", "resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-4.2.1.tgz", @@ -2159,6 +3706,13 @@ "node": ">=0.10.0" } }, + "node_modules/split-ca": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/split-ca/-/split-ca-1.0.1.tgz", + "integrity": "sha512-Q5thBSxp5t8WPTTJQS59LrGqOZqOsrhDGDVm8azCqIBjSBd7nd9o2PM+mDulQQkh8h//4U6hFZnc/mul8t5pWQ==", + "dev": true, + "license": "ISC" + }, "node_modules/split2": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", @@ -2168,6 +3722,46 @@ "node": ">= 10.x" } }, + "node_modules/ssh-remote-port-forward": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/ssh-remote-port-forward/-/ssh-remote-port-forward-1.0.4.tgz", + "integrity": "sha512-x0LV1eVDwjf1gmG7TTnfqIzf+3VPRz7vrNIjX6oYLbeCrf/PeVY6hkT68Mg+q02qXxQhrLjB0jfgvhevoCRmLQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/ssh2": "^0.5.48", + "ssh2": "^1.4.0" + } + }, + "node_modules/ssh-remote-port-forward/node_modules/@types/ssh2": { + "version": "0.5.52", + "resolved": "https://registry.npmjs.org/@types/ssh2/-/ssh2-0.5.52.tgz", + "integrity": "sha512-lbLLlXxdCZOSJMCInKH2+9V/77ET2J6NPQHpFI0kda61Dd1KglJs+fPQBchizmzYSOJBgdTajhPqBO1xxLywvg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*", + "@types/ssh2-streams": "*" + } + }, + "node_modules/ssh2": { + "version": "1.17.0", + "resolved": "https://registry.npmjs.org/ssh2/-/ssh2-1.17.0.tgz", + "integrity": "sha512-wPldCk3asibAjQ/kziWQQt1Wh3PgDFpC0XpwclzKcdT1vql6KeYxf5LIt4nlFkUeR8WuphYMKqUA56X4rjbfgQ==", + "dev": true, + "hasInstallScript": true, + "dependencies": { + "asn1": "^0.2.6", + "bcrypt-pbkdf": "^1.0.2" + }, + "engines": { + "node": ">=10.16.0" + }, + "optionalDependencies": { + "cpu-features": "~0.0.10", + "nan": "^2.23.0" + } + }, "node_modules/stackback": { "version": "0.0.2", "resolved": "https://registry.npmjs.org/stackback/-/stackback-0.0.2.tgz", @@ -2182,6 +3776,207 @@ "dev": true, "license": "MIT" }, + "node_modules/streamx": { + "version": "2.28.1", + "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.28.1.tgz", + "integrity": "sha512-zEzXb0s5Cds7tqMH6rhZ05lcJydCWiQPEwiNngVqzsxCc962vLY4Uw+mW7od8kDH258k2Uz/JrOkdIAAhSh9VA==", + "dev": true, + "license": "MIT", + "dependencies": { + "events-universal": "^1.0.0", + "fast-fifo": "^1.3.2", + "text-decoder": "^1.1.0" + } + }, + "node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "dev": true, + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, + "node_modules/string-width": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", + "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "eastasianwidth": "^0.2.0", + "emoji-regex": "^9.2.2", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/string-width-cjs": { + "name": "string-width", + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/string-width-cjs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/string-width-cjs/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true, + "license": "MIT" + }, + "node_modules/string-width-cjs/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.2.0.tgz", + "integrity": "sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^6.2.2" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/strip-ansi?sponsor=1" + } + }, + "node_modules/strip-ansi-cjs": { + "name": "strip-ansi", + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi-cjs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/tar-fs": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-3.1.3.tgz", + "integrity": "sha512-/hU4AXnIdZu+Gvl1pk0oI5f5HxWsCJRtY2aFaJdk9VvyL48DWU6iU5WAIPG+wIi1YvWA6eTJvIviP/tMAZZNwQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "pump": "^3.0.0", + "tar-stream": "^3.1.5" + }, + "optionalDependencies": { + "bare-fs": "^4.0.1", + "bare-path": "^3.0.0" + } + }, + "node_modules/tar-stream": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-3.2.1.tgz", + "integrity": "sha512-nqsEO8zLZJvrOMdEwkA0QdCLFbetHMn95Zqu4fKwX+hkaTWJPZZOrxx/PwtxoK0MMGQmBQNRW3CPs8IFYQz4cQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "b4a": "^1.6.4", + "bare-fs": "^4.5.5", + "fast-fifo": "^1.2.0", + "streamx": "^2.15.0" + } + }, + "node_modules/teex": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/teex/-/teex-1.0.1.tgz", + "integrity": "sha512-eYE6iEI62Ni1H8oIa7KlDU6uQBtqr4Eajni3wX7rpfXD8ysFx8z0+dri+KWEPWpBsxXfxu58x/0jvTVT1ekOSg==", + "dev": true, + "license": "MIT", + "dependencies": { + "streamx": "^2.12.5" + } + }, + "node_modules/testcontainers": { + "version": "12.1.0", + "resolved": "https://registry.npmjs.org/testcontainers/-/testcontainers-12.1.0.tgz", + "integrity": "sha512-YjDLqIITuhGLMnM10yhg3oV6lIG5IMpz1R1DPBZoOOks83q7i7IVpeSWRTiyl7roozjiyLmwIoLK/KY8OnZmIA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@balena/dockerignore": "^1.0.2", + "@types/dockerode": "^4.0.1", + "archiver": "^7.0.1", + "async-lock": "^1.4.1", + "byline": "^5.0.0", + "debug": "^4.4.3", + "docker-compose": "^1.4.2", + "dockerode": "^5.0.1", + "get-port": "^5.1.1", + "proper-lockfile": "^4.1.2", + "properties-reader": "^3.0.1", + "ssh-remote-port-forward": "^1.0.4", + "tar-fs": "^3.1.3", + "tmp": "^0.2.7", + "undici": "^8.9.0" + }, + "engines": { + "node": ">= 22.22" + } + }, + "node_modules/text-decoder": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/text-decoder/-/text-decoder-1.2.7.tgz", + "integrity": "sha512-vlLytXkeP4xvEq2otHeJfSQIRyWxo/oZGEbXrtEEF9Hnmrdly59sUbzZ/QgyWuLYHctCHxFF4tRQZNQ9k60ExQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "b4a": "^1.6.4" + } + }, "node_modules/thread-stream": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/thread-stream/-/thread-stream-4.2.0.tgz", @@ -2244,6 +4039,16 @@ "node": ">=14.0.0" } }, + "node_modules/tmp": { + "version": "0.2.7", + "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.7.tgz", + "integrity": "sha512-e0votIpp4Uo2AJYSzVHV6xCcawuiez3DzqDAbrTc3YxBkplN6e+dM13ZeIcZnDg/QpSuU2zfZ3rzwY8ukEnaXw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14.14" + } + }, "node_modules/toad-cache": { "version": "3.7.1", "resolved": "https://registry.npmjs.org/toad-cache/-/toad-cache-3.7.1.tgz", @@ -2272,6 +4077,13 @@ "fsevents": "~2.3.3" } }, + "node_modules/tweetnacl": { + "version": "0.14.5", + "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", + "integrity": "sha512-KXXFFdAbFXY4geFIwoyNK+f5Z1b7swfXABfL7HXCmoIWMKU3dmS26672A4EeQtDzLKy7SXmfBu51JolvEKwtGA==", + "dev": true, + "license": "Unlicense" + }, "node_modules/typescript": { "version": "5.9.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", @@ -2286,12 +4098,29 @@ "node": ">=14.17" } }, + "node_modules/undici": { + "version": "8.10.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-8.10.0.tgz", + "integrity": "sha512-HvltHd7avK13QIw/oLe4qoOLyoVSoafqJ2jYOrtMRBkbYT31eiBQ8O0ehRKZiEZCMEyLFQNIADpgCWC5fALvYQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=22.19.0" + } + }, "node_modules/undici-types": { "version": "7.18.2", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.18.2.tgz", "integrity": "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w==", "license": "MIT" }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", + "dev": true, + "license": "MIT" + }, "node_modules/vite": { "version": "5.4.21", "resolved": "https://registry.npmjs.org/vite/-/vite-5.4.21.tgz", @@ -2871,6 +4700,22 @@ } } }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dev": true, + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, "node_modules/why-is-node-running": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/why-is-node-running/-/why-is-node-running-2.3.0.tgz", @@ -2888,6 +4733,111 @@ "node": ">=8" } }, + "node_modules/wrap-ansi": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", + "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^6.1.0", + "string-width": "^5.0.1", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrap-ansi-cjs": { + "name": "wrap-ansi", + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true, + "license": "MIT" + }, + "node_modules/wrap-ansi-cjs/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "dev": true, + "license": "ISC" + }, "node_modules/xtend": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", @@ -2897,6 +4847,16 @@ "node": ">=0.4" } }, + "node_modules/y18n": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=10" + } + }, "node_modules/yaml": { "version": "2.9.0", "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.9.0.tgz", @@ -2912,6 +4872,95 @@ "url": "https://github.com/sponsors/eemeli" } }, + "node_modules/yargs": { + "version": "17.7.3", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.3.tgz", + "integrity": "sha512-GZtjxm/J/4TSxuL3FNYjCmLktBTnIw/rVmKSIyKeYAZpmJB2ig9VauCC5xsa82GNKVKDAqpOn3KVzNt0zmrU0g==", + "dev": true, + "license": "MIT", + "dependencies": { + "cliui": "^8.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.3", + "y18n": "^5.0.5", + "yargs-parser": "^21.1.1" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/yargs-parser": { + "version": "21.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/yargs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/yargs/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true, + "license": "MIT" + }, + "node_modules/yargs/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/yargs/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/zip-stream": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/zip-stream/-/zip-stream-6.0.1.tgz", + "integrity": "sha512-zK7YHHz4ZXpW89AHXUPbQVGKI7uvkd3hzusTdotCg1UxyaVtg0zFJSTfW/Dq5f7OBBVnq6cZIaC8Ti4hb6dtCA==", + "dev": true, + "license": "MIT", + "dependencies": { + "archiver-utils": "^5.0.0", + "compress-commons": "^6.0.2", + "readable-stream": "^4.0.0" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/zod": { "version": "4.4.3", "resolved": "https://registry.npmjs.org/zod/-/zod-4.4.3.tgz", diff --git a/backend/package.json b/backend/package.json index 359c7605..a71ece93 100644 --- a/backend/package.json +++ b/backend/package.json @@ -6,6 +6,7 @@ "dev": "tsx watch src/server.ts", "prebuild": "node scripts/clean-dist.mjs", "build": "tsc -p tsconfig.json", + "catalog:migrate": "node dist/catalog/migrate.js", "test": "vitest run", "start": "node dist/server.js", "test:schema-v3-verifier": "python3 -I -B scripts/test_revision_state_policy.py && node --test scripts/verify-workspace-descriptor-files.test.mjs scripts/revision-state-policy.test.mjs" @@ -16,12 +17,14 @@ "@fastify/rate-limit": "11.2.0", "@types/pg": "^8.20.3", "fastify": "^5.0.0", + "kysely": "^0.29.5", "openid-client": "6.8.5", "pg": "^8.22.0", "yaml": "^2.9.0", "zod": "^4.4.3" }, "devDependencies": { + "@testcontainers/postgresql": "^12.1.0", "@types/node": "24.13.3", "tsx": "^4.19.0", "typescript": "^5.6.0", diff --git a/backend/scripts/ssh-askpass.mjs b/backend/scripts/ssh-askpass.mjs new file mode 100755 index 00000000..241d2645 --- /dev/null +++ b/backend/scripts/ssh-askpass.mjs @@ -0,0 +1,6 @@ +#!/usr/bin/env node +import { readFileSync } from "node:fs"; + +const path = process.env.THT_SSH_PASSPHRASE_FILE; +if (!path) process.exit(1); +process.stdout.write(readFileSync(path)); diff --git a/backend/src/app.ts b/backend/src/app.ts index f7fdfe5f..453527db 100644 --- a/backend/src/app.ts +++ b/backend/src/app.ts @@ -37,6 +37,17 @@ import { supportsSessionRuntime } from "./workspaces/bindings.js"; import { resolveRuntimeBindingsWithWorkspaceSecrets } from "./workspaces/secret-requirements.js"; import type { WorkspaceDescriptor } from "./workspaces/schema.js"; import { WorkspaceSecretStore } from "./workspaces/secret-store.js"; +import { createCatalogRepository } from "./catalog/repository.js"; +import type { CatalogRepository } from "./catalog/types.js"; +import { CatalogService } from "./catalog/service.js"; +import { catalogDatabaseRoutes } from "./routes/catalog-databases.js"; +import { CatalogOperationCoordinator } from "./catalog/operation-coordinator.js"; +import { ConcreteCatalogPostgresAccess, type CatalogPostgresAccess } from "./catalog/postgres-access.js"; +import { CatalogTableService } from "./catalog/table-service.js"; +import { catalogTableRoutes } from "./routes/catalog-tables.js"; +import { ConcreteCatalogSchemaIntrospector, type CatalogSchemaIntrospector } from "./catalog/schema-introspector.js"; +import { CatalogSyncWorker } from "./catalog/sync-worker.js"; +import { catalogSchemaRoutes } from "./routes/catalog-schema.js"; export interface BuildAppDeps { thtRunner?: ThtRunner; @@ -49,6 +60,13 @@ export interface BuildAppDeps { workspaceRegistry?: WorkspaceRegistry; workspaceDiagnoser?: WorkspaceDiagnoser; workspaceSecretStore?: WorkspaceSecretStore; + catalogRepository?: CatalogRepository; + catalogService?: CatalogService; + catalogPostgresAccess?: CatalogPostgresAccess; + catalogTableService?: CatalogTableService; + catalogSchemaIntrospector?: CatalogSchemaIntrospector; + catalogSyncWorker?: CatalogSyncWorker; + catalogOperationCoordinator?: CatalogOperationCoordinator; workspaceRuntimeSupport?: (workspace: WorkspaceDescriptor) => boolean; maintenanceBarrier?: MaintenanceBarrier; piManagement?: PiManagementService; @@ -121,6 +139,37 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc const mgr = deps?.mgr ?? new PiProcessManager(config, deps?.spawnFn ? { spawnFn: deps.spawnFn } : undefined); const hub = deps?.hub ?? new SseHub(); const workspaceRegistry = deps?.workspaceRegistry ?? new WorkspaceRegistry(config.workspaceRegistry); + const catalogRepository = deps?.catalogRepository ?? createCatalogRepository(config.catalogDatabase); + const catalogOperationCoordinator = deps?.catalogOperationCoordinator ?? new CatalogOperationCoordinator(); + const catalogPostgresAccess = deps?.catalogPostgresAccess ?? new ConcreteCatalogPostgresAccess( + workspaceSecretStore, + { connectTimeoutMs: config.workspaceDiagnosticTimeoutMs }, + ); + const catalogService = deps?.catalogService ?? new CatalogService( + catalogRepository, + workspaceRegistry, + workspaceSecretStore, + config.workspaceRegistry.secretRoots, + config.workspaceDiagnosticTimeoutMs, + catalogPostgresAccess, + catalogOperationCoordinator, + ); + const catalogTableService = deps?.catalogTableService ?? new CatalogTableService(catalogRepository); + const catalogSchemaIntrospector = deps?.catalogSchemaIntrospector ?? new ConcreteCatalogSchemaIntrospector( + catalogPostgresAccess, + workspaceSecretStore, + ); + const catalogSyncWorker = deps?.catalogSyncWorker ?? new CatalogSyncWorker( + catalogRepository, + catalogSchemaIntrospector, + catalogOperationCoordinator, + config.catalogSyncTimeoutMs, + ); + app.addHook("onReady", async () => { await catalogSyncWorker.initialize(); }); + if (!deps?.catalogRepository && catalogRepository.close) { + app.addHook("onClose", async () => { await catalogRepository.close?.(); }); + } + app.addHook("onClose", async () => { await catalogSyncWorker.stop(); }); const workspaceDiagnoser = deps?.workspaceDiagnoser ?? createProductionWorkspaceDiagnoser(config.workspaceDiagnosticTimeoutMs, undefined, { internalQdrantUrl: config.internalQdrantUrl, @@ -334,6 +383,9 @@ export function buildApp(config: AppConfig, deps?: BuildAppDeps): FastifyInstanc authDiagnoser, secretStore: workspaceSecretStore, }); + catalogDatabaseRoutes(app, { repository: catalogRepository, service: catalogService, operations: catalogOperationCoordinator }); + catalogTableRoutes(app, { repository: catalogRepository, service: catalogTableService }); + catalogSchemaRoutes(app, { repository: catalogRepository, worker: catalogSyncWorker }); settingsRoutes(app, { cfg: config, listModels, getSettings }); piManagementRoutes(app, { service: piManagement }); diff --git a/backend/src/auth/config.ts b/backend/src/auth/config.ts index 47d77a54..82429cde 100644 --- a/backend/src/auth/config.ts +++ b/backend/src/auth/config.ts @@ -38,7 +38,7 @@ const MAX_MAPPED_GROUPS = 128; const ROLES = ["user", "admin"] as const; export const PERMISSION_CATALOG: readonly Permission[] = [ "session.use", "session.read_all", "session.manage_all", "settings.manage", - "workspace.manage", "workspace.secrets.manage", "pi.manage", "auth.diagnostics.read", + "workspace.manage", "workspace.secrets.manage", "database.manage", "pi.manage", "auth.diagnostics.read", ]; const invalid = (): Error => new Error("authentication configuration is invalid"); diff --git a/backend/src/auth/session-store.ts b/backend/src/auth/session-store.ts index 6e5fa08a..a7d52379 100644 --- a/backend/src/auth/session-store.ts +++ b/backend/src/auth/session-store.ts @@ -33,7 +33,7 @@ const EMPTY_HKDF_SALT = Buffer.alloc(0); const ROLES = ["user", "admin"] as const; const PERMISSIONS = [ "session.use", "session.read_all", "session.manage_all", "settings.manage", - "workspace.manage", "workspace.secrets.manage", "pi.manage", "auth.diagnostics.read", + "workspace.manage", "workspace.secrets.manage", "database.manage", "pi.manage", "auth.diagnostics.read", ] as const satisfies readonly Permission[]; const invalid = (): Error => new Error("auth_session_store_invalid"); diff --git a/backend/src/auth/types.ts b/backend/src/auth/types.ts index 3cc8955b..d5d74db1 100644 --- a/backend/src/auth/types.ts +++ b/backend/src/auth/types.ts @@ -5,7 +5,7 @@ export type Role = "user" | "admin"; export type Permission = | "session.use" | "session.read_all" | "session.manage_all" | "settings.manage" | "workspace.manage" | "workspace.secrets.manage" - | "pi.manage" | "auth.diagnostics.read"; + | "database.manage" | "pi.manage" | "auth.diagnostics.read"; export interface AuthenticationSessionConfig { regularTtlSeconds: number; diff --git a/backend/src/catalog/memory-repository.ts b/backend/src/catalog/memory-repository.ts new file mode 100644 index 00000000..c4d85d8c --- /dev/null +++ b/backend/src/catalog/memory-repository.ts @@ -0,0 +1,692 @@ +import { randomUUID } from "node:crypto"; +import { + CatalogConflictError, + CatalogConnectorError, + type CatalogColumn, + type CatalogRelationship, + type CatalogSchemaDiff, + type CatalogSyncCounts, + type CatalogSyncEvent, + type CatalogSyncRun, + type CatalogSyncRunUpdate, + type CatalogSyncScope, + type CatalogTable, + type CatalogRepository, + type DatabaseConfigurationInput, + type DatabaseTestResult, + type ObservedCatalogTable, + type ObservedSchemaSnapshot, + type TableSyncRepositoryResult, + type WorkspaceDatabase, +} from "./types.js"; + +function clone(value: WorkspaceDatabase): WorkspaceDatabase { + return structuredClone(value); +} + +/** Deterministic repository used by route tests and local contract consumers. */ +export class MemoryCatalogRepository implements CatalogRepository { + private readonly records = new Map(); + private readonly tables = new Map(); + private readonly columns = new Map(); + private readonly relationships = new Map(); + private readonly syncRuns = new Map(); + private readonly syncEvents = new Map(); + + async list(): Promise { + return [...this.records.values()].sort((a, b) => a.workspaceId.localeCompare(b.workspaceId)).map(clone); + } + async get(id: string): Promise { + const value = this.records.get(id); + return value ? clone(value) : undefined; + } + async getByWorkspace(workspaceId: string): Promise { + const value = [...this.records.values()].find((record) => record.workspaceId === workspaceId); + return value ? clone(value) : undefined; + } + async create(input: DatabaseConfigurationInput): Promise { + if ([...this.records.values()].some((record) => record.workspaceId === input.workspaceId)) { + throw new CatalogConflictError("Workspace database already exists"); + } + const now = new Date().toISOString(); + const record: WorkspaceDatabase = { + id: randomUUID(), + ...structuredClone(input), + version: 1, + createdAt: now, + updatedAt: now, + connectionStatus: "untested", + }; + this.records.set(record.id, record); + return clone(record); + } + async update(id: string, expectedVersion: number, input: DatabaseConfigurationInput): Promise { + const current = this.records.get(id); + if (!current || current.version !== expectedVersion) return undefined; + if ([...this.records.values()].some((record) => record.id !== id && record.workspaceId === input.workspaceId)) { + throw new CatalogConflictError("Workspace database already exists"); + } + const updated: WorkspaceDatabase = { + ...current, + ...structuredClone(input), + version: current.version + 1, + updatedAt: new Date().toISOString(), + connectionStatus: "untested", + testedVersion: undefined, + lastTestedAt: undefined, + lastErrorCode: undefined, + lastErrorMessage: undefined, + schemaSyncedVersion: undefined, + schemaSyncedAt: undefined, + }; + this.records.set(id, updated); + return clone(updated); + } + async recordTest(id: string, expectedVersion: number, result: DatabaseTestResult): Promise { + const current = this.records.get(id); + if (!current || current.version !== expectedVersion) return undefined; + const updated = { + ...current, + connectionStatus: result.connectionStatus, + testedVersion: result.testedVersion, + lastTestedAt: result.lastTestedAt, + lastErrorCode: result.errorCode, + lastErrorMessage: result.errorMessage, + }; + this.records.set(id, updated); + return clone(updated); + } + async touch(id: string, expectedVersion: number): Promise { + const current = this.records.get(id); + if (!current || current.version !== expectedVersion) return undefined; + return await this.update(id, expectedVersion, { + workspaceId: current.workspaceId, + engine: current.engine, + databaseName: current.databaseName, + schema: current.schema, + binding: current.binding, + }); + } + async delete(id: string, expectedVersion: number): Promise { + const current = this.records.get(id); + if (!current || current.version !== expectedVersion) return false; + for (const [tableId, table] of this.tables) { + if (table.databaseId === id) { + this.tables.delete(tableId); + for (const [columnId, column] of this.columns) if (column.tableId === tableId) this.columns.delete(columnId); + } + } + for (const [relationshipId, relationship] of this.relationships) { + if (relationship.databaseId === id) this.relationships.delete(relationshipId); + } + return this.records.delete(id); + } + async listTables(databaseId: string): Promise { + return [...this.tables.values()] + .filter((table) => table.databaseId === databaseId) + .sort((a, b) => a.name.localeCompare(b.name)) + .map((table) => structuredClone(table)); + } + async getTable(databaseId: string, tableId: string): Promise { + const table = this.tables.get(tableId); + return table?.databaseId === databaseId ? structuredClone(table) : undefined; + } + async updateTableDescription( + databaseId: string, + tableId: string, + expectedVersion: number, + description: string | null, + ): Promise { + const current = this.tables.get(tableId); + if (!current || current.databaseId !== databaseId || current.version !== expectedVersion) return undefined; + const updated = { + ...current, + description, + version: current.version + 1, + updatedAt: new Date().toISOString(), + }; + this.tables.set(tableId, updated); + return structuredClone(updated); + } + async updateTableMetadata( + databaseId: string, + tableId: string, + expectedVersion: number, + description: string | null, + generatedDescription: string | null, + ): Promise { + const current = this.tables.get(tableId); + if (!current || current.databaseId !== databaseId || current.version !== expectedVersion) return undefined; + const updated = { + ...current, description, generatedDescription, version: current.version + 1, + updatedAt: new Date().toISOString(), + }; + this.tables.set(tableId, updated); + return structuredClone(updated); + } + + async listColumns(databaseId: string, tableId: string): Promise { + const table = this.tables.get(tableId); + if (!table || table.databaseId !== databaseId) return []; + return [...this.columns.values()].filter((column) => column.tableId === tableId) + .sort((a, b) => a.ordinalPosition - b.ordinalPosition).map((column) => structuredClone(column)); + } + + async getColumn(databaseId: string, tableId: string, columnId: string): Promise { + const table = this.tables.get(tableId); + const column = this.columns.get(columnId); + return table?.databaseId === databaseId && column?.tableId === tableId ? structuredClone(column) : undefined; + } + + async updateColumnMetadata( + databaseId: string, + tableId: string, + columnId: string, + expectedVersion: number, + description: string | null, + generatedDescription: string | null, + ): Promise { + const current = await this.getColumn(databaseId, tableId, columnId); + if (!current || current.version !== expectedVersion) return undefined; + const updated = { ...current, description, generatedDescription, version: current.version + 1, updatedAt: new Date().toISOString() }; + this.columns.set(columnId, updated); + return structuredClone(updated); + } + + async listRelationships(databaseId: string): Promise { + return [...this.relationships.values()].filter((relationship) => relationship.databaseId === databaseId) + .sort((a, b) => `${a.sourceTableName}.${a.constraintName}`.localeCompare(`${b.sourceTableName}.${b.constraintName}`)) + .map((relationship) => structuredClone(relationship)); + } + + async planSchemaSync( + databaseId: string, + scope: CatalogSyncScope, + tableIds: readonly string[], + snapshot: ObservedSchemaSnapshot, + ): Promise { + this.assertSnapshotCapability(scope, snapshot); + const tables = await this.listTables(databaseId); + const selectedTableIds = new Set(tableIds); + const selectedTables = scope === "columns" && selectedTableIds.size > 0 + ? tables.filter((table) => selectedTableIds.has(table.id)) + : tables; + const observedTables = new Set(snapshot.tables.map((table) => table.name)); + const observedColumns = new Set(snapshot.columns.map((column) => `${column.tableName}\u0000${column.name}`)); + const observedRelationships = new Set( + snapshot.relationships.map((relationship) => `${relationship.sourceTableName}\u0000${relationship.constraintName}`), + ); + const deletedTableIds = new Set(tables.filter((table) => !observedTables.has(table.name)).map((table) => table.id)); + + return { + deletedTables: scope === "tables" || scope === "all" + ? tables.filter((table) => !observedTables.has(table.name)).map((table) => table.name).sort() + : [], + deletedColumns: scope === "tables" || scope === "columns" || scope === "all" + ? [...this.columns.values()] + .filter((column) => scope === "tables" ? deletedTableIds.has(column.tableId) : selectedTables.some((table) => table.id === column.tableId)) + .filter((column) => { + const table = tables.find((candidate) => candidate.id === column.tableId); + return table && (scope === "tables" || !observedColumns.has(`${table.name}\u0000${column.name}`)); + }) + .map((column) => ({ + tableName: tables.find((table) => table.id === column.tableId)?.name ?? "", + columnName: column.name, + })) + .sort((a, b) => `${a.tableName}.${a.columnName}`.localeCompare(`${b.tableName}.${b.columnName}`)) + : [], + deletedRelationships: scope === "tables" || scope === "relationships" || scope === "all" + ? [...this.relationships.values()] + .filter((relationship) => relationship.databaseId === databaseId) + .filter((relationship) => scope === "tables" + ? deletedTableIds.has(relationship.sourceTableId) || deletedTableIds.has(relationship.targetTableId) + : !observedRelationships.has(`${relationship.sourceTableName}\u0000${relationship.constraintName}`)) + .map((relationship) => ({ + sourceTableName: relationship.sourceTableName, + constraintName: relationship.constraintName, + })) + .sort((a, b) => `${a.sourceTableName}.${a.constraintName}`.localeCompare(`${b.sourceTableName}.${b.constraintName}`)) + : [], + }; + } + + async applySchemaSync( + databaseId: string, + expectedDatabaseVersion: number, + scope: CatalogSyncScope, + tableIds: readonly string[], + snapshot: ObservedSchemaSnapshot, + ): Promise { + const database = this.records.get(databaseId); + if (!database || database.version !== expectedDatabaseVersion) return undefined; + this.assertSnapshotCapability(scope, snapshot); + + const tableBackup = structuredClone([...this.tables.entries()]); + const columnBackup = structuredClone([...this.columns.entries()]); + const relationshipBackup = structuredClone([...this.relationships.entries()]); + const now = new Date().toISOString(); + let created = 0; + let updated = 0; + let deleted = 0; + + try { + if (scope === "tables" || scope === "all") { + const observedByName = new Map(snapshot.tables.map((table) => [table.name, table])); + const existing = await this.listTables(databaseId); + for (const table of existing) { + const observed = observedByName.get(table.name); + if (!observed) { + this.deleteTable(table.id); + deleted += 1; + } else { + const changed = table.sourceComment !== observed.sourceComment; + this.tables.set(table.id, { + ...table, + sourceComment: observed.sourceComment, + lastSyncedDatabaseVersion: expectedDatabaseVersion, + lastSyncedAt: now, + version: changed ? table.version + 1 : table.version, + updatedAt: changed ? now : table.updatedAt, + }); + if (changed) updated += 1; + observedByName.delete(table.name); + } + } + for (const observed of observedByName.values()) { + const id = randomUUID(); + this.tables.set(id, { + id, + databaseId, + name: observed.name, + sourceComment: observed.sourceComment, + description: null, + generatedDescription: null, + lastSyncedDatabaseVersion: expectedDatabaseVersion, + lastSyncedAt: now, + version: 1, + createdAt: now, + updatedAt: now, + }); + created += 1; + } + } + + if (scope === "columns" || scope === "all") { + const currentTables = await this.listTables(databaseId); + const selectedIds = scope === "all" || tableIds.length === 0 + ? new Set(currentTables.map((table) => table.id)) + : new Set(tableIds); + const selectedTables = currentTables.filter((table) => selectedIds.has(table.id)); + if (scope === "columns" && selectedTables.length !== selectedIds.size) { + throw new CatalogConnectorError("One or more selected tables no longer exist"); + } + const selectedNames = new Set(selectedTables.map((table) => table.name)); + const tableByName = new Map(selectedTables.map((table) => [table.name, table])); + const observedByKey = new Map( + snapshot.columns + .filter((column) => selectedNames.has(column.tableName)) + .map((column) => [`${column.tableName}\u0000${column.name}`, column]), + ); + for (const column of [...this.columns.values()].filter((candidate) => selectedIds.has(candidate.tableId))) { + const table = selectedTables.find((candidate) => candidate.id === column.tableId); + if (!table) continue; + const key = `${table.name}\u0000${column.name}`; + const observed = observedByKey.get(key); + if (!observed) { + this.deleteColumn(column.id); + deleted += 1; + } else { + const changed = column.ordinalPosition !== observed.ordinalPosition + || column.dataType !== observed.dataType + || column.isNullable !== observed.isNullable + || column.defaultExpression !== observed.defaultExpression + || column.primaryKeyPosition !== observed.primaryKeyPosition + || column.sourceComment !== observed.sourceComment; + this.columns.set(column.id, { + ...column, + ordinalPosition: observed.ordinalPosition, + dataType: observed.dataType, + isNullable: observed.isNullable, + defaultExpression: observed.defaultExpression, + primaryKeyPosition: observed.primaryKeyPosition, + isPrimaryKey: observed.primaryKeyPosition !== null, + sourceComment: observed.sourceComment, + lastSyncedDatabaseVersion: expectedDatabaseVersion, + lastSyncedAt: now, + version: changed ? column.version + 1 : column.version, + updatedAt: changed ? now : column.updatedAt, + }); + if (changed) updated += 1; + observedByKey.delete(key); + } + } + for (const observed of observedByKey.values()) { + const table = tableByName.get(observed.tableName); + if (!table) continue; + const id = randomUUID(); + this.columns.set(id, { + id, + tableId: table.id, + name: observed.name, + ordinalPosition: observed.ordinalPosition, + dataType: observed.dataType, + isNullable: observed.isNullable, + defaultExpression: observed.defaultExpression, + primaryKeyPosition: observed.primaryKeyPosition, + isPrimaryKey: observed.primaryKeyPosition !== null, + isForeignKey: false, + foreignKeyCount: 0, + sourceComment: observed.sourceComment, + description: null, + generatedDescription: null, + lastSyncedDatabaseVersion: expectedDatabaseVersion, + lastSyncedAt: now, + version: 1, + createdAt: now, + updatedAt: now, + }); + created += 1; + } + } + + if (scope === "relationships" || scope === "all") { + const tables = await this.listTables(databaseId); + const tableByName = new Map(tables.map((table) => [table.name, table])); + const existing = [...this.relationships.values()].filter((relationship) => relationship.databaseId === databaseId); + const existingByKey = new Map(existing.map((relationship) => [`${relationship.sourceTableName}\u0000${relationship.constraintName}`, relationship])); + const observedKeys = new Set(snapshot.relationships.map((relationship) => `${relationship.sourceTableName}\u0000${relationship.constraintName}`)); + for (const relationship of existing) { + if (!observedKeys.has(`${relationship.sourceTableName}\u0000${relationship.constraintName}`)) { + this.relationships.delete(relationship.id); + deleted += 1; + } + } + for (const observed of snapshot.relationships) { + const sourceTable = tableByName.get(observed.sourceTableName); + const targetTable = tableByName.get(observed.targetTableName); + if (!sourceTable || !targetTable) { + throw new CatalogConnectorError(`Relationship ${observed.constraintName} refers to an unknown table`); + } + const pairs = observed.columns.map((pair) => { + const source = [...this.columns.values()].find((column) => column.tableId === sourceTable.id && column.name === pair.sourceColumnName); + const target = [...this.columns.values()].find((column) => column.tableId === targetTable.id && column.name === pair.targetColumnName); + if (!source || !target) { + throw new CatalogConnectorError(`Relationship ${observed.constraintName} refers to an unknown column`); + } + return { + position: pair.position, + sourceColumnId: source.id, + sourceColumnName: source.name, + targetColumnId: target.id, + targetColumnName: target.name, + }; + }).sort((a, b) => a.position - b.position); + const key = `${observed.sourceTableName}\u0000${observed.constraintName}`; + const current = existingByKey.get(key); + const comparable = current && JSON.stringify({ + target: current.targetTableName, + update: current.updateRule, + delete: current.deleteRule, + deferrable: current.deferrable, + deferred: current.initiallyDeferred, + columns: current.columns.map((pair) => [pair.position, pair.sourceColumnName, pair.targetColumnName]), + }); + const nextComparable = JSON.stringify({ + target: observed.targetTableName, + update: observed.updateRule, + delete: observed.deleteRule, + deferrable: observed.deferrable, + deferred: observed.initiallyDeferred, + columns: pairs.map((pair) => [pair.position, pair.sourceColumnName, pair.targetColumnName]), + }); + const id = current?.id ?? randomUUID(); + this.relationships.set(id, { + id, + databaseId, + constraintName: observed.constraintName, + sourceTableId: sourceTable.id, + sourceTableName: sourceTable.name, + targetTableId: targetTable.id, + targetTableName: targetTable.name, + updateRule: observed.updateRule, + deleteRule: observed.deleteRule, + deferrable: observed.deferrable, + initiallyDeferred: observed.initiallyDeferred, + columns: pairs, + lastSyncedDatabaseVersion: expectedDatabaseVersion, + lastSyncedAt: now, + createdAt: current?.createdAt ?? now, + updatedAt: comparable === nextComparable ? (current?.updatedAt ?? now) : now, + }); + if (!current) created += 1; + else if (comparable !== nextComparable) updated += 1; + } + this.refreshForeignKeyFlags(databaseId); + } + + if (scope === "all") { + this.records.set(databaseId, { ...database, schemaSyncedVersion: expectedDatabaseVersion, schemaSyncedAt: now }); + } + return { + tables: (await this.listTables(databaseId)).length, + columns: [...this.columns.values()].filter((column) => this.tables.get(column.tableId)?.databaseId === databaseId).length, + relationships: (await this.listRelationships(databaseId)).length, + created, + updated, + deleted, + }; + } catch (error) { + this.tables.clear(); + this.columns.clear(); + this.relationships.clear(); + for (const [id, table] of tableBackup) this.tables.set(id, table); + for (const [id, column] of columnBackup) this.columns.set(id, column); + for (const [id, relationship] of relationshipBackup) this.relationships.set(id, relationship); + throw error; + } + } + + async createSyncRun( + databaseId: string, + scope: CatalogSyncScope, + tableIds: readonly string[], + requestedDatabaseVersion: number, + ): Promise { + const activeStates = new Set(["queued", "running", "awaiting_confirmation", "applying"]); + if ([...this.syncRuns.values()].some((run) => run.databaseId === databaseId && activeStates.has(run.state))) { + throw new CatalogConflictError("A schema synchronization is already active for this database"); + } + const now = new Date().toISOString(); + const run: CatalogSyncRun = { + id: randomUUID(), databaseId, scope, tableIds: [...tableIds], state: "queued", phase: "queued", + requestedDatabaseVersion, observedSnapshot: null, plannedDiff: null, confirmationToken: null, + counts: {}, errorCode: null, errorMessage: null, cancelRequested: false, + createdAt: now, startedAt: null, updatedAt: now, finishedAt: null, heartbeatAt: null, + leaseOwner: null, leaseExpiresAt: null, + }; + this.syncRuns.set(run.id, run); + return structuredClone(run); + } + + async getSyncRun(runId: string): Promise { + const run = this.syncRuns.get(runId); + return run ? structuredClone(run) : undefined; + } + + async claimSyncRun(runId: string, workerId: string, leaseExpiresAt: string): Promise { + const run = this.syncRuns.get(runId); + if (!run || run.state !== "queued") return undefined; + if (run.leaseOwner && run.leaseOwner !== workerId && run.leaseExpiresAt && run.leaseExpiresAt > new Date().toISOString()) { + return undefined; + } + return await this.updateSyncRun(runId, { + state: "running", + startedAt: run.startedAt ?? new Date().toISOString(), + heartbeatAt: new Date().toISOString(), + leaseOwner: workerId, + leaseExpiresAt, + }); + } + + async listSyncRuns(databaseId: string, limit = 20): Promise { + return [...this.syncRuns.values()].filter((run) => run.databaseId === databaseId) + .sort((a, b) => b.createdAt.localeCompare(a.createdAt)).slice(0, limit).map((run) => structuredClone(run)); + } + + async updateSyncRun(runId: string, update: CatalogSyncRunUpdate): Promise { + const current = this.syncRuns.get(runId); + if (!current) return undefined; + const updated = { ...current, ...structuredClone(update), updatedAt: new Date().toISOString() }; + this.syncRuns.set(runId, updated); + return structuredClone(updated); + } + + async requestSyncRunCancellation(runId: string): Promise { + const run = this.syncRuns.get(runId); + if (!run) return undefined; + if (!["queued", "running", "awaiting_confirmation"].includes(run.state)) return structuredClone(run); + return await this.updateSyncRun(runId, { cancelRequested: true }); + } + + async appendSyncEvent( + runId: string, + level: CatalogSyncEvent["level"], + eventType: string, + message: string, + data: Record = {}, + ): Promise { + const events = this.syncEvents.get(runId) ?? []; + const event: CatalogSyncEvent = { + id: [...this.syncEvents.values()].reduce((count, values) => count + values.length, 0) + 1, + runId, sequence: events.length + 1, level, eventType, message, data: structuredClone(data), + createdAt: new Date().toISOString(), + }; + events.push(event); + this.syncEvents.set(runId, events); + return structuredClone(event); + } + + async listSyncEvents(runId: string, afterSequence = 0): Promise { + return (this.syncEvents.get(runId) ?? []).filter((event) => event.sequence > afterSequence).map((event) => structuredClone(event)); + } + + async pruneSyncEvents(before: string): Promise { + for (const [runId, events] of this.syncEvents) { + this.syncEvents.set(runId, events.filter((event) => event.createdAt >= before)); + } + } + + async interruptActiveSyncRuns(): Promise { + for (const run of this.syncRuns.values()) { + if (["queued", "running", "awaiting_confirmation", "applying"].includes(run.state)) { + await this.updateSyncRun(run.id, { + state: "interrupted", phase: "completed", finishedAt: new Date().toISOString(), + errorCode: "SYNC_INTERRUPTED", errorMessage: "Synchronization was interrupted by a service restart", + }); + } + } + } + + async reconcileTables( + databaseId: string, + expectedDatabaseVersion: number, + observed: readonly ObservedCatalogTable[], + confirmedDeletedNames: readonly string[], + ): Promise { + const database = this.records.get(databaseId); + if (!database || database.version !== expectedDatabaseVersion) return undefined; + const existing = await this.listTables(databaseId); + const observedNames = new Set(observed.map((table) => table.name)); + const deletedNames = existing.filter((table) => !observedNames.has(table.name)).map((table) => table.name).sort(); + const confirmation = [...new Set(confirmedDeletedNames)].sort(); + if (deletedNames.length > 0 && JSON.stringify(deletedNames) !== JSON.stringify(confirmation)) { + return { kind: "confirmation_required", deletedNames }; + } + + const byName = new Map(existing.map((table) => [table.name, table])); + let createdCount = 0; + let updatedCount = 0; + for (const observedTable of observed) { + const current = byName.get(observedTable.name); + const now = new Date().toISOString(); + if (!current) { + const created: CatalogTable = { + id: randomUUID(), + databaseId, + name: observedTable.name, + sourceComment: observedTable.sourceComment, + description: null, + generatedDescription: null, + lastSyncedDatabaseVersion: expectedDatabaseVersion, + lastSyncedAt: now, + version: 1, + createdAt: now, + updatedAt: now, + }; + this.tables.set(created.id, created); + createdCount += 1; + } else if (current.sourceComment !== observedTable.sourceComment) { + this.tables.set(current.id, { + ...current, + sourceComment: observedTable.sourceComment, + version: current.version + 1, + updatedAt: now, + }); + updatedCount += 1; + } + } + for (const table of existing) { + if (deletedNames.includes(table.name)) this.deleteTable(table.id); + } + return { + kind: "applied", + createdCount, + updatedCount, + deletedCount: deletedNames.length, + tables: await this.listTables(databaseId), + }; + } + + private assertSnapshotCapability(scope: CatalogSyncScope, snapshot: ObservedSchemaSnapshot): void { + const required = scope === "all" ? ["tables", "columns", "relationships"] as const : [scope] as const; + for (const capability of required) { + if (snapshot.capabilities[capability] !== "available") { + throw new CatalogConnectorError(`Schema introspection capability '${capability}' is unavailable`); + } + } + } + + private deleteTable(tableId: string): void { + this.tables.delete(tableId); + for (const column of [...this.columns.values()]) if (column.tableId === tableId) this.deleteColumn(column.id); + for (const relationship of [...this.relationships.values()]) { + if (relationship.sourceTableId === tableId || relationship.targetTableId === tableId) { + this.relationships.delete(relationship.id); + } + } + } + + private deleteColumn(columnId: string): void { + this.columns.delete(columnId); + for (const relationship of [...this.relationships.values()]) { + if (relationship.columns.some((pair) => pair.sourceColumnId === columnId || pair.targetColumnId === columnId)) { + this.relationships.delete(relationship.id); + } + } + } + + private refreshForeignKeyFlags(databaseId: string): void { + const counts = new Map(); + for (const relationship of this.relationships.values()) { + if (relationship.databaseId !== databaseId) continue; + for (const pair of relationship.columns) counts.set(pair.sourceColumnId, (counts.get(pair.sourceColumnId) ?? 0) + 1); + } + for (const column of this.columns.values()) { + if (this.tables.get(column.tableId)?.databaseId !== databaseId) continue; + const foreignKeyCount = counts.get(column.id) ?? 0; + this.columns.set(column.id, { ...column, isForeignKey: foreignKeyCount > 0, foreignKeyCount }); + } + } + + async available(): Promise { return true; } +} diff --git a/backend/src/catalog/migrate.ts b/backend/src/catalog/migrate.ts new file mode 100644 index 00000000..a11be877 --- /dev/null +++ b/backend/src/catalog/migrate.ts @@ -0,0 +1,51 @@ +import { CamelCasePlugin, Kysely, PostgresDialect } from "kysely"; +import { Migrator, type MigrationProvider } from "kysely/migration"; +import { Pool } from "pg"; +import { readFile } from "node:fs/promises"; +import type { CatalogDatabase } from "./repository.js"; +import * as initialMigration from "./migrations/001_workspace_databases.js"; +import * as catalogTablesMigration from "./migrations/002_catalog_tables.js"; +import * as catalogSchemaSyncMigration from "./migrations/003_catalog_schema_sync.js"; +import * as catalogRuntimeSequencePrivilegesMigration from "./migrations/004_catalog_runtime_sequence_privileges.js"; + +const connectionString = process.env.THT_CATALOG_MIGRATOR_DATABASE_URL; +const host = process.env.THT_CATALOG_DB_HOST; +const database = process.env.THT_CATALOG_DB_NAME; +const user = process.env.THT_CATALOG_MIGRATOR_USER; +const passwordFile = process.env.THT_CATALOG_MIGRATOR_PASSWORD_FILE; +if (!connectionString && (!host || !database || !user || !passwordFile)) { + throw new Error("catalog migrator database configuration is required"); +} +const pool = new Pool(connectionString ? { connectionString, max: 1 } : { + host, + port: Number(process.env.THT_CATALOG_DB_PORT ?? 5432), + database, + user, + password: async () => (await readFile(passwordFile!, "utf8")).trim(), + max: 1, +}); + +const db = new Kysely({ + dialect: new PostgresDialect({ pool }), + plugins: [new CamelCasePlugin()], +}); +const provider: MigrationProvider = { + async getMigrations() { + return { + "001_workspace_databases": initialMigration, + "002_catalog_tables": catalogTablesMigration, + "003_catalog_schema_sync": catalogSchemaSyncMigration, + "004_catalog_runtime_sequence_privileges": catalogRuntimeSequencePrivilegesMigration, + }; + }, +}; + +try { + const result = await new Migrator({ db, provider }).migrateToLatest(); + for (const item of result.results ?? []) { + process.stdout.write(`${item.migrationName}: ${item.status}\n`); + } + if (result.error) throw result.error; +} finally { + await db.destroy(); +} diff --git a/backend/src/catalog/migrations/001_workspace_databases.ts b/backend/src/catalog/migrations/001_workspace_databases.ts new file mode 100644 index 00000000..1c4745b2 --- /dev/null +++ b/backend/src/catalog/migrations/001_workspace_databases.ts @@ -0,0 +1,58 @@ +import { sql, type Kysely } from "kysely"; +import type { CatalogDatabase } from "../repository.js"; + +export async function up(db: Kysely): Promise { + await db.schema.createTable("workspace_databases") + .addColumn("id", "uuid", (column) => column.primaryKey()) + .addColumn("workspace_id", "text", (column) => column.notNull().unique()) + .addColumn("engine", "text", (column) => column.notNull()) + .addColumn("database_name", "text", (column) => column.notNull()) + .addColumn("schema_name", "text", (column) => column.notNull()) + .addColumn("version", "integer", (column) => column.notNull().defaultTo(1)) + .addColumn("created_at", "timestamptz", (column) => column.notNull().defaultTo(sql`now()`)) + .addColumn("updated_at", "timestamptz", (column) => column.notNull().defaultTo(sql`now()`)) + .addCheckConstraint("workspace_databases_engine_check", sql`engine = 'postgres'`) + .addCheckConstraint("workspace_databases_version_check", sql`version > 0`) + .execute(); + + await db.schema.createTable("database_bindings") + .addColumn("database_id", "uuid", (column) => column.primaryKey() + .references("workspace_databases.id").onDelete("cascade")) + .addColumn("transport", "text", (column) => column.notNull()) + .addColumn("host", "text") + .addColumn("port", "integer") + .addColumn("username", "text") + .addColumn("base_url", "text") + .addColumn("rest_path", "text") + .addColumn("rest_auth", "text") + .addColumn("tls_servername", "text") + .addColumn("ssh_host", "text") + .addColumn("ssh_port", "integer") + .addColumn("ssh_username", "text") + .addColumn("ssh_target_host", "text") + .addColumn("ssh_target_port", "integer") + .addColumn("connection_status", "text", (column) => column.notNull().defaultTo("untested")) + .addColumn("tested_version", "integer") + .addColumn("last_tested_at", "timestamptz") + .addColumn("last_error_code", "text") + .addColumn("last_error_message", "text") + .addColumn("updated_at", "timestamptz", (column) => column.notNull().defaultTo(sql`now()`)) + .addCheckConstraint("database_bindings_transport_check", sql`transport in ('postgres_direct', 'rest_api', 'ssh_tunnel')`) + .addCheckConstraint("database_bindings_status_check", sql`connection_status in ('untested', 'reachable', 'failed')`) + .addCheckConstraint("database_bindings_port_check", sql`port is null or port between 1 and 65535`) + .addCheckConstraint("database_bindings_ssh_port_check", sql`ssh_port is null or ssh_port between 1 and 65535`) + .addCheckConstraint("database_bindings_ssh_target_port_check", sql`ssh_target_port is null or ssh_target_port between 1 and 65535`) + .addCheckConstraint("database_bindings_transport_fields_check", sql` + (transport = 'postgres_direct' and host is not null and port is not null and username is not null) + or (transport = 'rest_api' and base_url is not null and rest_path is not null and rest_auth is not null) + or (transport = 'ssh_tunnel' and username is not null and ssh_host is not null + and ssh_port is not null and ssh_username is not null and ssh_target_host is not null + and ssh_target_port is not null) + `) + .execute(); +} + +export async function down(db: Kysely): Promise { + await db.schema.dropTable("database_bindings").execute(); + await db.schema.dropTable("workspace_databases").execute(); +} diff --git a/backend/src/catalog/migrations/002_catalog_tables.ts b/backend/src/catalog/migrations/002_catalog_tables.ts new file mode 100644 index 00000000..3766f799 --- /dev/null +++ b/backend/src/catalog/migrations/002_catalog_tables.ts @@ -0,0 +1,27 @@ +import { sql, type Kysely } from "kysely"; +import type { CatalogDatabase } from "../repository.js"; + +export async function up(db: Kysely): Promise { + await db.schema.createTable("catalog_tables") + .addColumn("id", "uuid", (column) => column.primaryKey()) + .addColumn("database_id", "uuid", (column) => column.notNull() + .references("workspace_databases.id").onDelete("cascade")) + .addColumn("name", "text", (column) => column.notNull()) + .addColumn("source_comment", "text") + .addColumn("description", "text") + .addColumn("generated_description", "text") + .addColumn("version", "integer", (column) => column.notNull().defaultTo(1)) + .addColumn("created_at", "timestamptz", (column) => column.notNull().defaultTo(sql`now()`)) + .addColumn("updated_at", "timestamptz", (column) => column.notNull().defaultTo(sql`now()`)) + .addUniqueConstraint("catalog_tables_database_name_key", ["database_id", "name"]) + .addCheckConstraint("catalog_tables_name_check", sql`char_length(name) between 1 and 128`) + .addCheckConstraint("catalog_tables_version_check", sql`version > 0`) + .execute(); + + await db.schema.createIndex("catalog_tables_database_id_idx") + .on("catalog_tables").column("database_id").execute(); +} + +export async function down(db: Kysely): Promise { + await db.schema.dropTable("catalog_tables").execute(); +} diff --git a/backend/src/catalog/migrations/003_catalog_schema_sync.ts b/backend/src/catalog/migrations/003_catalog_schema_sync.ts new file mode 100644 index 00000000..db0fbc2f --- /dev/null +++ b/backend/src/catalog/migrations/003_catalog_schema_sync.ts @@ -0,0 +1,136 @@ +import { sql, type Kysely } from "kysely"; +import type { CatalogDatabase } from "../repository.js"; + +export async function up(db: Kysely): Promise { + await db.schema.alterTable("workspace_databases") + .addColumn("schema_synced_version", "integer") + .addColumn("schema_synced_at", "timestamptz") + .execute(); + + await db.schema.alterTable("catalog_tables") + .addColumn("last_synced_database_version", "integer") + .addColumn("last_synced_at", "timestamptz") + .execute(); + + await db.schema.createTable("catalog_columns") + .addColumn("id", "uuid", (column) => column.primaryKey()) + .addColumn("table_id", "uuid", (column) => column.notNull() + .references("catalog_tables.id").onDelete("cascade")) + .addColumn("name", "text", (column) => column.notNull()) + .addColumn("ordinal_position", "integer", (column) => column.notNull()) + .addColumn("data_type", "text", (column) => column.notNull()) + .addColumn("is_nullable", "boolean", (column) => column.notNull()) + .addColumn("default_expression", "text") + .addColumn("primary_key_position", "integer") + .addColumn("source_comment", "text") + .addColumn("description", "text") + .addColumn("generated_description", "text") + .addColumn("last_synced_database_version", "integer") + .addColumn("last_synced_at", "timestamptz") + .addColumn("version", "integer", (column) => column.notNull().defaultTo(1)) + .addColumn("created_at", "timestamptz", (column) => column.notNull().defaultTo(sql`now()`)) + .addColumn("updated_at", "timestamptz", (column) => column.notNull().defaultTo(sql`now()`)) + .addUniqueConstraint("catalog_columns_table_name_key", ["table_id", "name"]) + .addCheckConstraint("catalog_columns_name_check", sql`char_length(name) between 1 and 128`) + .addCheckConstraint("catalog_columns_ordinal_check", sql`ordinal_position > 0`) + .addCheckConstraint("catalog_columns_pk_position_check", sql`primary_key_position is null or primary_key_position > 0`) + .addCheckConstraint("catalog_columns_version_check", sql`version > 0`) + .execute(); + await db.schema.createIndex("catalog_columns_table_id_idx") + .on("catalog_columns").column("table_id").execute(); + + await db.schema.createTable("catalog_relationships") + .addColumn("id", "uuid", (column) => column.primaryKey()) + .addColumn("database_id", "uuid", (column) => column.notNull() + .references("workspace_databases.id").onDelete("cascade")) + .addColumn("constraint_name", "text", (column) => column.notNull()) + .addColumn("source_table_id", "uuid", (column) => column.notNull() + .references("catalog_tables.id").onDelete("cascade")) + .addColumn("target_table_id", "uuid", (column) => column.notNull() + .references("catalog_tables.id").onDelete("cascade")) + .addColumn("update_rule", "text", (column) => column.notNull()) + .addColumn("delete_rule", "text", (column) => column.notNull()) + .addColumn("deferrable", "boolean", (column) => column.notNull().defaultTo(false)) + .addColumn("initially_deferred", "boolean", (column) => column.notNull().defaultTo(false)) + .addColumn("last_synced_database_version", "integer") + .addColumn("last_synced_at", "timestamptz") + .addColumn("created_at", "timestamptz", (column) => column.notNull().defaultTo(sql`now()`)) + .addColumn("updated_at", "timestamptz", (column) => column.notNull().defaultTo(sql`now()`)) + .addUniqueConstraint("catalog_relationships_source_constraint_key", ["source_table_id", "constraint_name"]) + .execute(); + await db.schema.createIndex("catalog_relationships_database_id_idx") + .on("catalog_relationships").column("database_id").execute(); + + await db.schema.createTable("catalog_relationship_columns") + .addColumn("relationship_id", "uuid", (column) => column.notNull() + .references("catalog_relationships.id").onDelete("cascade")) + .addColumn("position", "integer", (column) => column.notNull()) + .addColumn("source_column_id", "uuid", (column) => column.notNull() + .references("catalog_columns.id").onDelete("cascade")) + .addColumn("target_column_id", "uuid", (column) => column.notNull() + .references("catalog_columns.id").onDelete("cascade")) + .addPrimaryKeyConstraint("catalog_relationship_columns_pkey", ["relationship_id", "position"]) + .addCheckConstraint("catalog_relationship_columns_position_check", sql`position > 0`) + .execute(); + + await db.schema.createTable("catalog_sync_runs") + .addColumn("id", "uuid", (column) => column.primaryKey()) + .addColumn("database_id", "uuid", (column) => column.notNull() + .references("workspace_databases.id").onDelete("cascade")) + .addColumn("scope", "text", (column) => column.notNull()) + .addColumn("table_ids", "jsonb", (column) => column.notNull().defaultTo(sql`'[]'::jsonb`)) + .addColumn("state", "text", (column) => column.notNull()) + .addColumn("phase", "text", (column) => column.notNull()) + .addColumn("requested_database_version", "integer", (column) => column.notNull()) + .addColumn("observed_snapshot", "jsonb") + .addColumn("planned_diff", "jsonb") + .addColumn("confirmation_token", "text") + .addColumn("counts", "jsonb", (column) => column.notNull().defaultTo(sql`'{}'::jsonb`)) + .addColumn("error_code", "text") + .addColumn("error_message", "text") + .addColumn("cancel_requested", "boolean", (column) => column.notNull().defaultTo(false)) + .addColumn("created_at", "timestamptz", (column) => column.notNull().defaultTo(sql`now()`)) + .addColumn("started_at", "timestamptz") + .addColumn("updated_at", "timestamptz", (column) => column.notNull().defaultTo(sql`now()`)) + .addColumn("finished_at", "timestamptz") + .addColumn("heartbeat_at", "timestamptz") + .addColumn("lease_owner", "text") + .addColumn("lease_expires_at", "timestamptz") + .execute(); + await db.schema.createIndex("catalog_sync_runs_database_created_idx") + .on("catalog_sync_runs").columns(["database_id", "created_at"]).execute(); + await sql`CREATE UNIQUE INDEX catalog_sync_runs_one_active_per_database + ON catalog_sync_runs (database_id) + WHERE state IN ('queued', 'running', 'awaiting_confirmation', 'applying')`.execute(db); + + await db.schema.createTable("catalog_sync_events") + .addColumn("id", "bigserial", (column) => column.primaryKey()) + .addColumn("run_id", "uuid", (column) => column.notNull() + .references("catalog_sync_runs.id").onDelete("cascade")) + .addColumn("sequence", "integer", (column) => column.notNull()) + .addColumn("level", "text", (column) => column.notNull()) + .addColumn("event_type", "text", (column) => column.notNull()) + .addColumn("message", "text", (column) => column.notNull()) + .addColumn("data", "jsonb", (column) => column.notNull().defaultTo(sql`'{}'::jsonb`)) + .addColumn("created_at", "timestamptz", (column) => column.notNull().defaultTo(sql`now()`)) + .addUniqueConstraint("catalog_sync_events_run_sequence_key", ["run_id", "sequence"]) + .execute(); + await db.schema.createIndex("catalog_sync_events_run_id_idx") + .on("catalog_sync_events").columns(["run_id", "sequence"]).execute(); +} + +export async function down(db: Kysely): Promise { + await db.schema.dropTable("catalog_sync_events").execute(); + await db.schema.dropTable("catalog_sync_runs").execute(); + await db.schema.dropTable("catalog_relationship_columns").execute(); + await db.schema.dropTable("catalog_relationships").execute(); + await db.schema.dropTable("catalog_columns").execute(); + await db.schema.alterTable("catalog_tables") + .dropColumn("last_synced_database_version") + .dropColumn("last_synced_at") + .execute(); + await db.schema.alterTable("workspace_databases") + .dropColumn("schema_synced_version") + .dropColumn("schema_synced_at") + .execute(); +} diff --git a/backend/src/catalog/migrations/004_catalog_runtime_sequence_privileges.ts b/backend/src/catalog/migrations/004_catalog_runtime_sequence_privileges.ts new file mode 100644 index 00000000..59fb4e71 --- /dev/null +++ b/backend/src/catalog/migrations/004_catalog_runtime_sequence_privileges.ts @@ -0,0 +1,29 @@ +import { type Kysely, sql } from "kysely"; +import type { CatalogDatabase } from "../repository.js"; + +/** + * `catalog_sync_events.id` is the first catalog-owned identity sequence. + * Table default privileges do not cover sequences, and without USAGE the + * runtime can create a run but cannot append its first event. + */ +export async function up(db: Kysely): Promise { + await sql`DO $catalog_privileges$ + BEGIN + IF EXISTS (SELECT 1 FROM pg_catalog.pg_roles WHERE rolname = 'thothii_catalog_runtime') THEN + EXECUTE 'GRANT USAGE, SELECT ON SEQUENCE catalog_sync_events_id_seq TO thothii_catalog_runtime'; + EXECUTE 'ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT USAGE, SELECT ON SEQUENCES TO thothii_catalog_runtime'; + END IF; + END + $catalog_privileges$`.execute(db); +} + +export async function down(db: Kysely): Promise { + await sql`DO $catalog_privileges$ + BEGIN + IF EXISTS (SELECT 1 FROM pg_catalog.pg_roles WHERE rolname = 'thothii_catalog_runtime') THEN + EXECUTE 'ALTER DEFAULT PRIVILEGES IN SCHEMA public REVOKE USAGE, SELECT ON SEQUENCES FROM thothii_catalog_runtime'; + EXECUTE 'REVOKE USAGE, SELECT ON SEQUENCE catalog_sync_events_id_seq FROM thothii_catalog_runtime'; + END IF; + END + $catalog_privileges$`.execute(db); +} diff --git a/backend/src/catalog/operation-coordinator.ts b/backend/src/catalog/operation-coordinator.ts new file mode 100644 index 00000000..1ec7160a --- /dev/null +++ b/backend/src/catalog/operation-coordinator.ts @@ -0,0 +1,28 @@ +import { CatalogOperationInProgressError } from "./types.js"; + +/** Serializes connection tests and schema synchronization for each catalog database. */ +export class CatalogOperationCoordinator { + private readonly active = new Set(); + + reserve(databaseId: string): () => void { + if (this.active.has(databaseId)) { + throw new CatalogOperationInProgressError("A database operation is already in progress"); + } + this.active.add(databaseId); + let released = false; + return () => { + if (released) return; + released = true; + this.active.delete(databaseId); + }; + } + + async run(databaseId: string, operation: () => Promise): Promise { + const release = this.reserve(databaseId); + try { + return await operation(); + } finally { + release(); + } + } +} diff --git a/backend/src/catalog/postgres-access.ts b/backend/src/catalog/postgres-access.ts new file mode 100644 index 00000000..0a493477 --- /dev/null +++ b/backend/src/catalog/postgres-access.ts @@ -0,0 +1,258 @@ +import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process"; +import { readFile } from "node:fs/promises"; +import { fileURLToPath } from "node:url"; +import { Duplex } from "node:stream"; +import { setTimeout as delay } from "node:timers/promises"; +import { Client, type ClientConfig } from "pg"; +import type { WorkspaceSecretStore } from "../workspaces/secret-store.js"; +import { CATALOG_SECRET_IDS } from "./secrets.js"; +import { CatalogConnectorError, type WorkspaceDatabase } from "./types.js"; + +export interface CatalogDatabaseClient { + query(sql: string, values: readonly unknown[]): Promise<{ rows: Array> }>; + end(): Promise; +} + +export interface CatalogPostgresAccess { + connect(database: WorkspaceDatabase, signal: AbortSignal): Promise; +} + +type SpawnSsh = ( + command: string, + args: readonly string[], + options: { env: NodeJS.ProcessEnv }, +) => ChildProcessWithoutNullStreams; + +interface AccessDependencies { + createClient?: (config: ClientConfig) => Client; + spawnSsh?: SpawnSsh; + sshBinary?: string; + askpassPath?: string; + connectTimeoutMs?: number; +} + +function required(value: string | number | undefined): string | number { + if (value === undefined || value === "") throw new CatalogConnectorError("Database binding is incomplete"); + return value; +} + +function connectionSsl(ca: string | undefined, servername: string | undefined): ClientConfig["ssl"] { + if (!ca && !servername) return false; + return { + ...(ca ? { ca } : {}), + ...(servername ? { servername } : {}), + rejectUnauthorized: true, + }; +} + +export function buildSshArguments(input: { + sshHost: string; + sshPort: number; + sshUsername: string; + targetHost: string; + targetPort: number; + privateKeyFile: string; + knownHostsFile: string; + passphraseFile?: string; + connectTimeoutMs: number; +}): string[] { + const batchMode = input.passphraseFile ? "no" : "yes"; + return [ + "-F", "/dev/null", + "-T", + "-o", `BatchMode=${batchMode}`, + "-o", "StrictHostKeyChecking=yes", + "-o", `UserKnownHostsFile=${input.knownHostsFile}`, + "-o", "GlobalKnownHostsFile=/dev/null", + "-o", "IdentitiesOnly=yes", + "-o", "IdentityAgent=none", + "-o", `IdentityFile=${input.privateKeyFile}`, + "-o", "PreferredAuthentications=publickey", + "-o", "PasswordAuthentication=no", + "-o", "KbdInteractiveAuthentication=no", + "-o", "ConnectionAttempts=1", + "-o", `ConnectTimeout=${Math.max(1, Math.ceil(input.connectTimeoutMs / 1_000))}`, + "-o", "ServerAliveInterval=5", + "-o", "ServerAliveCountMax=1", + "-o", "NumberOfPasswordPrompts=1", + "-o", "RequestTTY=no", + "-o", "LogLevel=ERROR", + "-p", String(input.sshPort), + "-W", `${input.targetHost}:${input.targetPort}`, + "--", `${input.sshUsername}@${input.sshHost}`, + ]; +} + +async function stopChild(child: ChildProcessWithoutNullStreams): Promise { + if (child.exitCode !== null || child.signalCode !== null) return; + child.kill("SIGTERM"); + await Promise.race([ + new Promise((resolve) => child.once("exit", () => resolve())), + delay(500).then(() => undefined), + ]); + if (child.exitCode === null && child.signalCode === null) child.kill("SIGKILL"); +} + +function sshDuplex(child: ChildProcessWithoutNullStreams): Duplex { + let ended = false; + let stream: Duplex; + const forward = () => { + let chunk: Buffer | string | null; + while ((chunk = child.stdout.read() as Buffer | string | null) !== null) { + if (!stream.push(chunk)) break; + } + }; + const finish = () => { + if (ended) return; + ended = true; + stream.push(null); + }; + const fail = (error: Error) => stream.destroy(error); + stream = new Duplex({ + read: forward, + write: (chunk, encoding, callback) => child.stdin.write(chunk, encoding, callback), + final: (callback) => child.stdin.end(callback), + destroy: (error, callback) => { + child.stdout.off("readable", forward); + child.stdout.off("end", finish); + child.stdout.off("error", fail); + child.stdin.off("error", fail); + callback(error); + }, + }); + child.stdout.on("readable", forward); + child.stdout.once("end", finish); + child.stdout.once("error", fail); + child.stdin.once("error", fail); + return stream; +} + +/** + * Deep connection module for direct and SSH-forwarded PostgreSQL access. It owns secret leases, + * TLS, OpenSSH lifecycle, abort propagation, and pg cleanup behind one connect interface. + */ +export class ConcreteCatalogPostgresAccess implements CatalogPostgresAccess { + private readonly createClient: (config: ClientConfig) => Client; + private readonly spawnSsh: SpawnSsh; + private readonly sshBinary: string; + private readonly askpassPath: string; + private readonly connectTimeoutMs: number; + + constructor( + private readonly secretStore: WorkspaceSecretStore, + dependencies: AccessDependencies = {}, + ) { + this.createClient = dependencies.createClient ?? ((config) => new Client(config)); + this.spawnSsh = dependencies.spawnSsh ?? ((command, args, options) => ( + spawn(command, [...args], { ...options, stdio: ["pipe", "pipe", "pipe"] }) + )); + this.sshBinary = dependencies.sshBinary ?? process.env.THT_SSH_BIN ?? "ssh"; + this.askpassPath = dependencies.askpassPath + ?? process.env.THT_SSH_ASKPASS_BIN + ?? fileURLToPath(new URL("../../scripts/ssh-askpass.mjs", import.meta.url)); + this.connectTimeoutMs = dependencies.connectTimeoutMs ?? 5_000; + } + + async connect(database: WorkspaceDatabase, signal: AbortSignal): Promise { + if (database.binding.transport === "rest_api") { + throw new CatalogConnectorError("REST is not a PostgreSQL wire binding"); + } + const ids: string[] = [CATALOG_SECRET_IDS.password, CATALOG_SECRET_IDS.tlsCa]; + if (database.binding.transport === "ssh_tunnel") { + ids.push( + CATALOG_SECRET_IDS.sshPrivateKey, + CATALOG_SECRET_IDS.sshPrivateKeyPassphrase, + CATALOG_SECRET_IDS.sshKnownHosts, + ); + } + const materialized = this.secretStore.materialize(database.workspaceId, ids); + let child: ChildProcessWithoutNullStreams | undefined; + let stream: Duplex | undefined; + let client: Client | undefined; + let ended = false; + const close = async () => { + if (ended) return; + ended = true; + signal.removeEventListener("abort", abort); + if (client) await client.end().catch(() => undefined); + stream?.destroy(); + if (child) await stopChild(child); + materialized.release(); + }; + const abort = () => { void close(); }; + + try { + const passwordFile = materialized.files.get(CATALOG_SECRET_IDS.password); + if (!passwordFile) throw new CatalogConnectorError("Database password is not configured"); + const password = await readFile(passwordFile, "utf8"); + const tlsCaFile = materialized.files.get(CATALOG_SECRET_IDS.tlsCa); + const tlsCa = tlsCaFile ? await readFile(tlsCaFile, "utf8") : undefined; + let host: string; + let port: number; + + if (database.binding.transport === "ssh_tunnel") { + const privateKeyFile = materialized.files.get(CATALOG_SECRET_IDS.sshPrivateKey); + const knownHostsFile = materialized.files.get(CATALOG_SECRET_IDS.sshKnownHosts); + if (!privateKeyFile || !knownHostsFile) { + throw new CatalogConnectorError("SSH private key and known hosts are required"); + } + const passphraseFile = materialized.files.get(CATALOG_SECRET_IDS.sshPrivateKeyPassphrase); + host = String(required(database.binding.sshTargetHost)); + port = Number(required(database.binding.sshTargetPort)); + const args = buildSshArguments({ + sshHost: String(required(database.binding.sshHost)), + sshPort: Number(required(database.binding.sshPort)), + sshUsername: String(required(database.binding.sshUsername)), + targetHost: host, + targetPort: port, + privateKeyFile, + knownHostsFile, + passphraseFile, + connectTimeoutMs: this.connectTimeoutMs, + }); + child = this.spawnSsh(this.sshBinary, args, { + env: { + ...process.env, + LC_ALL: "C", + ...(passphraseFile ? { + DISPLAY: "thothii", + SSH_ASKPASS: this.askpassPath, + SSH_ASKPASS_REQUIRE: "force", + THT_SSH_PASSPHRASE_FILE: passphraseFile, + } : {}), + }, + }); + stream = sshDuplex(child); + child.once("error", () => stream?.destroy(new CatalogConnectorError("SSH process failed"))); + child.once("exit", (code) => { + if (!ended && code !== 0) stream?.destroy(new CatalogConnectorError("SSH tunnel failed")); + }); + child.stderr.on("data", () => undefined); + } else { + host = String(required(database.binding.host)); + port = Number(required(database.binding.port)); + } + + client = this.createClient({ + host, + port, + database: database.databaseName, + user: String(required(database.binding.username)), + password, + ssl: connectionSsl(tlsCa, database.binding.tlsServername), + connectionTimeoutMillis: this.connectTimeoutMs, + ...(stream ? { stream: () => stream } : {}), + }); + signal.addEventListener("abort", abort, { once: true }); + await client.connect(); + return { + query: async (sql, values) => await client!.query(sql, [...values]), + end: close, + }; + } catch (error) { + await close(); + if (error instanceof CatalogConnectorError) throw error; + throw new CatalogConnectorError("PostgreSQL connector failed"); + } + } +} diff --git a/backend/src/catalog/repository.ts b/backend/src/catalog/repository.ts new file mode 100644 index 00000000..4bd551d7 --- /dev/null +++ b/backend/src/catalog/repository.ts @@ -0,0 +1,1052 @@ +import { randomUUID } from "node:crypto"; +import { readFile } from "node:fs/promises"; +import { + CamelCasePlugin, + Kysely, + PostgresDialect, + sql, + type ColumnType, + type Generated, + type Selectable, + type Transaction, +} from "kysely"; +import { Pool } from "pg"; +import { + CatalogConflictError, + CatalogConnectorError, + CatalogUnavailableError, + type CatalogColumn, + type CatalogRelationship, + type CatalogSchemaDiff, + type CatalogSyncCounts, + type CatalogSyncEvent, + type CatalogSyncRun, + type CatalogSyncRunUpdate, + type CatalogSyncScope, + type CatalogTable, + type CatalogRepository, + type DatabaseBinding, + type DatabaseConfigurationInput, + type DatabaseTestResult, + type ObservedCatalogTable, + type ObservedSchemaSnapshot, + type TableSyncRepositoryResult, + type WorkspaceDatabase, +} from "./types.js"; + +type Timestamp = ColumnType; + +interface WorkspaceDatabaseTable { + id: string; + workspaceId: string; + engine: "postgres"; + databaseName: string; + schemaName: string; + version: Generated; + createdAt: Timestamp; + updatedAt: Timestamp; + schemaSyncedVersion: number | null; + schemaSyncedAt: Timestamp | null; +} + +interface DatabaseBindingTable { + databaseId: string; + transport: DatabaseBinding["transport"]; + host: string | null; + port: number | null; + username: string | null; + baseUrl: string | null; + restPath: string | null; + restAuth: DatabaseBinding["restAuth"] | null; + tlsServername: string | null; + sshHost: string | null; + sshPort: number | null; + sshUsername: string | null; + sshTargetHost: string | null; + sshTargetPort: number | null; + connectionStatus: WorkspaceDatabase["connectionStatus"]; + testedVersion: number | null; + lastTestedAt: Timestamp | null; + lastErrorCode: string | null; + lastErrorMessage: string | null; + updatedAt: Timestamp; +} + +interface CatalogTableTable { + id: string; + databaseId: string; + name: string; + sourceComment: string | null; + description: string | null; + generatedDescription: string | null; + lastSyncedDatabaseVersion: number | null; + lastSyncedAt: Timestamp | null; + version: Generated; + createdAt: Timestamp; + updatedAt: Timestamp; +} + +interface CatalogColumnTable { + id: string; + tableId: string; + name: string; + ordinalPosition: number; + dataType: string; + isNullable: boolean; + defaultExpression: string | null; + primaryKeyPosition: number | null; + sourceComment: string | null; + description: string | null; + generatedDescription: string | null; + lastSyncedDatabaseVersion: number | null; + lastSyncedAt: Timestamp | null; + version: Generated; + createdAt: Timestamp; + updatedAt: Timestamp; +} + +interface CatalogRelationshipTable { + id: string; + databaseId: string; + constraintName: string; + sourceTableId: string; + targetTableId: string; + updateRule: string; + deleteRule: string; + deferrable: boolean; + initiallyDeferred: boolean; + lastSyncedDatabaseVersion: number | null; + lastSyncedAt: Timestamp | null; + createdAt: Timestamp; + updatedAt: Timestamp; +} + +interface CatalogRelationshipColumnTable { + relationshipId: string; + position: number; + sourceColumnId: string; + targetColumnId: string; +} + +interface CatalogSyncRunTable { + id: string; + databaseId: string; + scope: CatalogSyncScope; + // node-postgres serializes JavaScript arrays as PostgreSQL arrays. The + // catalog column is JSONB, so inserts must cross the driver boundary as a + // JSON string while reads are decoded back to a string array. + tableIds: ColumnType; + state: CatalogSyncRun["state"]; + phase: CatalogSyncRun["phase"]; + requestedDatabaseVersion: number; + observedSnapshot: ObservedSchemaSnapshot | null; + plannedDiff: CatalogSchemaDiff | null; + confirmationToken: string | null; + counts: CatalogSyncCounts; + errorCode: string | null; + errorMessage: string | null; + cancelRequested: boolean; + createdAt: Timestamp; + startedAt: Timestamp | null; + updatedAt: Timestamp; + finishedAt: Timestamp | null; + heartbeatAt: Timestamp | null; + leaseOwner: string | null; + leaseExpiresAt: Timestamp | null; +} + +interface CatalogSyncEventTable { + id: Generated; + runId: string; + sequence: number; + level: CatalogSyncEvent["level"]; + eventType: string; + message: string; + data: Record; + createdAt: Timestamp; +} + +export interface CatalogDatabase { + workspaceDatabases: WorkspaceDatabaseTable; + databaseBindings: DatabaseBindingTable; + catalogTables: CatalogTableTable; + catalogColumns: CatalogColumnTable; + catalogRelationships: CatalogRelationshipTable; + catalogRelationshipColumns: CatalogRelationshipColumnTable; + catalogSyncRuns: CatalogSyncRunTable; + catalogSyncEvents: CatalogSyncEventTable; +} + +type DbOrTransaction = Kysely | Transaction; +type JoinedRow = Selectable & Selectable; + +function present(value: T | null): T | undefined { + return value === null ? undefined : value; +} + +function serialize(row: JoinedRow): WorkspaceDatabase { + return { + id: row.id, + workspaceId: row.workspaceId, + engine: row.engine, + databaseName: row.databaseName, + schema: row.schemaName, + version: row.version, + createdAt: new Date(row.createdAt).toISOString(), + updatedAt: new Date(row.updatedAt).toISOString(), + binding: { + transport: row.transport, + host: present(row.host), + port: present(row.port), + username: present(row.username), + baseUrl: present(row.baseUrl), + restPath: present(row.restPath), + restAuth: present(row.restAuth), + tlsServername: present(row.tlsServername), + sshHost: present(row.sshHost), + sshPort: present(row.sshPort), + sshUsername: present(row.sshUsername), + sshTargetHost: present(row.sshTargetHost), + sshTargetPort: present(row.sshTargetPort), + }, + connectionStatus: row.connectionStatus, + testedVersion: present(row.testedVersion), + lastTestedAt: row.lastTestedAt === null ? undefined : new Date(row.lastTestedAt).toISOString(), + lastErrorCode: present(row.lastErrorCode), + lastErrorMessage: present(row.lastErrorMessage), + schemaSyncedVersion: present(row.schemaSyncedVersion), + schemaSyncedAt: row.schemaSyncedAt == null ? undefined : new Date(row.schemaSyncedAt).toISOString(), + }; +} + +function serializeTable(row: Selectable): CatalogTable { + return { + id: row.id, + databaseId: row.databaseId, + name: row.name, + sourceComment: row.sourceComment, + description: row.description, + generatedDescription: row.generatedDescription, + lastSyncedDatabaseVersion: row.lastSyncedDatabaseVersion, + lastSyncedAt: row.lastSyncedAt === null ? null : new Date(row.lastSyncedAt).toISOString(), + version: row.version, + createdAt: new Date(row.createdAt).toISOString(), + updatedAt: new Date(row.updatedAt).toISOString(), + }; +} + +function serializeColumn(row: Selectable, foreignKeyCount = 0): CatalogColumn { + return { + id: row.id, + tableId: row.tableId, + name: row.name, + ordinalPosition: row.ordinalPosition, + dataType: row.dataType, + isNullable: row.isNullable, + defaultExpression: row.defaultExpression, + primaryKeyPosition: row.primaryKeyPosition, + isPrimaryKey: row.primaryKeyPosition !== null, + isForeignKey: foreignKeyCount > 0, + foreignKeyCount, + sourceComment: row.sourceComment, + description: row.description, + generatedDescription: row.generatedDescription, + lastSyncedDatabaseVersion: row.lastSyncedDatabaseVersion, + lastSyncedAt: row.lastSyncedAt === null ? null : new Date(row.lastSyncedAt).toISOString(), + version: row.version, + createdAt: new Date(row.createdAt).toISOString(), + updatedAt: new Date(row.updatedAt).toISOString(), + }; +} + +function serializeSyncRun(row: Selectable): CatalogSyncRun { + const stamp = (value: Date | string | null) => value === null ? null : new Date(value).toISOString(); + return { + ...row, + tableIds: row.tableIds ?? [], + counts: row.counts ?? {}, + createdAt: new Date(row.createdAt).toISOString(), + startedAt: stamp(row.startedAt), + updatedAt: new Date(row.updatedAt).toISOString(), + finishedAt: stamp(row.finishedAt), + heartbeatAt: stamp(row.heartbeatAt), + leaseExpiresAt: stamp(row.leaseExpiresAt), + }; +} + +function serializeSyncEvent(row: Selectable): CatalogSyncEvent { + return { ...row, id: Number(row.id), data: row.data ?? {}, createdAt: new Date(row.createdAt).toISOString() }; +} + +function bindingValues(databaseId: string, binding: DatabaseBinding) { + return { + databaseId, + transport: binding.transport, + host: binding.host ?? null, + port: binding.port ?? null, + username: binding.username ?? null, + baseUrl: binding.baseUrl ?? null, + restPath: binding.restPath ?? null, + restAuth: binding.restAuth ?? null, + tlsServername: binding.tlsServername ?? null, + sshHost: binding.sshHost ?? null, + sshPort: binding.sshPort ?? null, + sshUsername: binding.sshUsername ?? null, + sshTargetHost: binding.sshTargetHost ?? null, + sshTargetPort: binding.sshTargetPort ?? null, + }; +} + +async function selectOne(db: DbOrTransaction, id: string): Promise { + const row = await db.selectFrom("workspaceDatabases") + .innerJoin("databaseBindings", "databaseBindings.databaseId", "workspaceDatabases.id") + .selectAll() + .where("workspaceDatabases.id", "=", id) + .executeTakeFirst(); + return row ? serialize(row as JoinedRow) : undefined; +} + +export class KyselyCatalogRepository implements CatalogRepository { + constructor(private readonly db: Kysely) {} + + async list(): Promise { + const rows = await this.db.selectFrom("workspaceDatabases") + .innerJoin("databaseBindings", "databaseBindings.databaseId", "workspaceDatabases.id") + .selectAll() + .orderBy("workspaceDatabases.workspaceId") + .execute(); + return rows.map((row) => serialize(row as JoinedRow)); + } + + async get(id: string): Promise { + return await selectOne(this.db, id); + } + + async getByWorkspace(workspaceId: string): Promise { + const id = await this.db.selectFrom("workspaceDatabases").select("id") + .where("workspaceId", "=", workspaceId).executeTakeFirst(); + return id ? await this.get(id.id) : undefined; + } + + async create(input: DatabaseConfigurationInput): Promise { + try { + return await this.db.transaction().execute(async (trx) => { + const id = randomUUID(); + await trx.insertInto("workspaceDatabases").values({ + id, + workspaceId: input.workspaceId, + engine: input.engine, + databaseName: input.databaseName, + schemaName: input.schema, + }).execute(); + await trx.insertInto("databaseBindings").values({ + ...bindingValues(id, input.binding), + connectionStatus: "untested", + testedVersion: null, + lastTestedAt: null, + lastErrorCode: null, + lastErrorMessage: null, + }).execute(); + return (await selectOne(trx, id))!; + }); + } catch (error: any) { + if (error?.code === "23505") throw new CatalogConflictError("Workspace database already exists"); + throw error; + } + } + + async update(id: string, expectedVersion: number, input: DatabaseConfigurationInput): Promise { + return await this.db.transaction().execute(async (trx) => { + const changed = await trx.updateTable("workspaceDatabases").set({ + workspaceId: input.workspaceId, + engine: input.engine, + databaseName: input.databaseName, + schemaName: input.schema, + version: sql`version + 1`, + updatedAt: sql`now()`, + }).where("id", "=", id).where("version", "=", expectedVersion).returning("id").executeTakeFirst(); + if (!changed) return undefined; + await trx.updateTable("databaseBindings").set({ + ...bindingValues(id, input.binding), + connectionStatus: "untested", + testedVersion: null, + lastTestedAt: null, + lastErrorCode: null, + lastErrorMessage: null, + updatedAt: sql`now()`, + }).where("databaseId", "=", id).execute(); + return await selectOne(trx, id); + }); + } + + async recordTest(id: string, expectedVersion: number, result: DatabaseTestResult): Promise { + const version = await this.db.selectFrom("workspaceDatabases").select("version") + .where("id", "=", id).where("version", "=", expectedVersion).executeTakeFirst(); + if (!version) return undefined; + await this.db.updateTable("databaseBindings").set({ + connectionStatus: result.connectionStatus, + testedVersion: result.testedVersion, + lastTestedAt: result.lastTestedAt, + lastErrorCode: result.errorCode ?? null, + lastErrorMessage: result.errorMessage ?? null, + updatedAt: sql`now()`, + }).where("databaseId", "=", id).execute(); + return await this.get(id); + } + + async touch(id: string, expectedVersion: number): Promise { + const current = await this.get(id); + if (!current || current.version !== expectedVersion) return undefined; + return await this.update(id, expectedVersion, { + workspaceId: current.workspaceId, + engine: current.engine, + databaseName: current.databaseName, + schema: current.schema, + binding: current.binding, + }); + } + + async delete(id: string, expectedVersion: number): Promise { + const result = await this.db.deleteFrom("workspaceDatabases") + .where("id", "=", id).where("version", "=", expectedVersion).executeTakeFirst(); + return result.numDeletedRows === 1n; + } + + async listTables(databaseId: string): Promise { + const rows = await this.db.selectFrom("catalogTables").selectAll() + .where("databaseId", "=", databaseId).orderBy("name").execute(); + return rows.map(serializeTable); + } + + async getTable(databaseId: string, tableId: string): Promise { + const row = await this.db.selectFrom("catalogTables").selectAll() + .where("databaseId", "=", databaseId).where("id", "=", tableId).executeTakeFirst(); + return row ? serializeTable(row) : undefined; + } + + async updateTableDescription( + databaseId: string, + tableId: string, + expectedVersion: number, + description: string | null, + ): Promise { + const row = await this.db.updateTable("catalogTables").set({ + description, + version: sql`version + 1`, + updatedAt: sql`now()`, + }).where("databaseId", "=", databaseId) + .where("id", "=", tableId) + .where("version", "=", expectedVersion) + .returningAll() + .executeTakeFirst(); + return row ? serializeTable(row) : undefined; + } + + async updateTableMetadata( + databaseId: string, + tableId: string, + expectedVersion: number, + description: string | null, + generatedDescription: string | null, + ): Promise { + const row = await this.db.updateTable("catalogTables").set({ + description, + generatedDescription, + version: sql`version + 1`, + updatedAt: sql`now()`, + }).where("databaseId", "=", databaseId) + .where("id", "=", tableId) + .where("version", "=", expectedVersion) + .returningAll() + .executeTakeFirst(); + return row ? serializeTable(row) : undefined; + } + + async listColumns(databaseId: string, tableId: string): Promise { + const rows = await this.db.selectFrom("catalogColumns") + .innerJoin("catalogTables", "catalogTables.id", "catalogColumns.tableId") + .selectAll("catalogColumns") + .where("catalogTables.databaseId", "=", databaseId) + .where("catalogColumns.tableId", "=", tableId) + .orderBy("catalogColumns.ordinalPosition") + .execute(); + if (rows.length === 0) return []; + const counts = await this.db.selectFrom("catalogRelationshipColumns") + .select(["sourceColumnId", sql`count(*)::int`.as("count")]) + .where("sourceColumnId", "in", rows.map((row) => row.id)) + .groupBy("sourceColumnId") + .execute(); + const byColumn = new Map(counts.map((row) => [row.sourceColumnId, Number(row.count)])); + return rows.map((row) => serializeColumn(row, byColumn.get(row.id) ?? 0)); + } + + async getColumn(databaseId: string, tableId: string, columnId: string): Promise { + const row = await this.db.selectFrom("catalogColumns") + .innerJoin("catalogTables", "catalogTables.id", "catalogColumns.tableId") + .selectAll("catalogColumns") + .where("catalogTables.databaseId", "=", databaseId) + .where("catalogColumns.tableId", "=", tableId) + .where("catalogColumns.id", "=", columnId) + .executeTakeFirst(); + if (!row) return undefined; + const count = await this.db.selectFrom("catalogRelationshipColumns") + .select(sql`count(*)::int`.as("count")) + .where("sourceColumnId", "=", columnId) + .executeTakeFirst(); + return serializeColumn(row, Number(count?.count ?? 0)); + } + + async updateColumnMetadata( + databaseId: string, + tableId: string, + columnId: string, + expectedVersion: number, + description: string | null, + generatedDescription: string | null, + ): Promise { + const belongs = await this.db.selectFrom("catalogTables").select("id") + .where("id", "=", tableId).where("databaseId", "=", databaseId).executeTakeFirst(); + if (!belongs) return undefined; + const row = await this.db.updateTable("catalogColumns").set({ + description, + generatedDescription, + version: sql`version + 1`, + updatedAt: sql`now()`, + }).where("id", "=", columnId).where("tableId", "=", tableId) + .where("version", "=", expectedVersion).returningAll().executeTakeFirst(); + return row ? await this.getColumn(databaseId, tableId, row.id) : undefined; + } + + async listRelationships(databaseId: string): Promise { + const rows = await this.db.selectFrom("catalogRelationships as relationship") + .innerJoin("catalogTables as sourceTable", "sourceTable.id", "relationship.sourceTableId") + .innerJoin("catalogTables as targetTable", "targetTable.id", "relationship.targetTableId") + .select([ + "relationship.id", "relationship.databaseId", "relationship.constraintName", + "relationship.sourceTableId", "sourceTable.name as sourceTableName", + "relationship.targetTableId", "targetTable.name as targetTableName", + "relationship.updateRule", "relationship.deleteRule", "relationship.deferrable", + "relationship.initiallyDeferred", "relationship.lastSyncedDatabaseVersion", + "relationship.lastSyncedAt", "relationship.createdAt", "relationship.updatedAt", + ]) + .where("relationship.databaseId", "=", databaseId) + .orderBy("sourceTable.name").orderBy("relationship.constraintName") + .execute(); + if (rows.length === 0) return []; + const pairs = await this.db.selectFrom("catalogRelationshipColumns as pair") + .innerJoin("catalogColumns as sourceColumn", "sourceColumn.id", "pair.sourceColumnId") + .innerJoin("catalogColumns as targetColumn", "targetColumn.id", "pair.targetColumnId") + .select([ + "pair.relationshipId", "pair.position", "pair.sourceColumnId", + "sourceColumn.name as sourceColumnName", "pair.targetColumnId", + "targetColumn.name as targetColumnName", + ]) + .where("pair.relationshipId", "in", rows.map((row) => row.id)) + .orderBy("pair.relationshipId").orderBy("pair.position") + .execute(); + const byRelationship = new Map(); + for (const pair of pairs) { + const items = byRelationship.get(pair.relationshipId) ?? []; + items.push(pair); + byRelationship.set(pair.relationshipId, items); + } + return rows.map((row) => ({ + ...row, + columns: byRelationship.get(row.id) ?? [], + lastSyncedAt: row.lastSyncedAt === null ? null : new Date(row.lastSyncedAt).toISOString(), + createdAt: new Date(row.createdAt).toISOString(), + updatedAt: new Date(row.updatedAt).toISOString(), + })); + } + + async planSchemaSync( + databaseId: string, + scope: CatalogSyncScope, + tableIds: readonly string[], + snapshot: ObservedSchemaSnapshot, + ): Promise { + const tables = await this.db.selectFrom("catalogTables").select(["id", "name"]) + .where("databaseId", "=", databaseId).execute(); + const selected = new Set(tableIds); + const targetTables = scope === "columns" && selected.size > 0 + ? tables.filter((table) => selected.has(table.id)) + : tables; + const observedTableNames = new Set(snapshot.tables.map((table) => table.name)); + const deletedTables = scope === "tables" || scope === "all" + ? tables.filter((table) => !observedTableNames.has(table.name)).map((table) => table.name).sort() + : []; + const deletedTableIds = new Set(tables.filter((table) => deletedTables.includes(table.name)).map((table) => table.id)); + const deletedColumns: CatalogSchemaDiff["deletedColumns"] = []; + if (scope === "tables" || scope === "columns" || scope === "all") { + const columnTables = scope === "tables" ? tables.filter((table) => deletedTableIds.has(table.id)) : targetTables; + const columns = columnTables.length === 0 ? [] : await this.db.selectFrom("catalogColumns") + .select(["tableId", "name"]).where("tableId", "in", columnTables.map((table) => table.id)).execute(); + const tableNameById = new Map(columnTables.map((table) => [table.id, table.name])); + const observed = new Set(snapshot.columns.map((column) => `${column.tableName}\u0000${column.name}`)); + for (const column of columns) { + const tableName = tableNameById.get(column.tableId)!; + if (scope === "tables" || !observed.has(`${tableName}\u0000${column.name}`)) { + deletedColumns.push({ tableName, columnName: column.name }); + } + } + deletedColumns.sort((a, b) => `${a.tableName}.${a.columnName}`.localeCompare(`${b.tableName}.${b.columnName}`)); + } + const deletedRelationships: CatalogSchemaDiff["deletedRelationships"] = []; + if (scope === "tables" || scope === "relationships" || scope === "all") { + const relationships = await this.db.selectFrom("catalogRelationships as relationship") + .innerJoin("catalogTables as sourceTable", "sourceTable.id", "relationship.sourceTableId") + .select(["relationship.constraintName", "relationship.sourceTableId", "relationship.targetTableId", "sourceTable.name as sourceTableName"]) + .where("relationship.databaseId", "=", databaseId).execute(); + const observed = new Set(snapshot.relationships.map((relationship) => + `${relationship.sourceTableName}\u0000${relationship.constraintName}`)); + for (const relationship of relationships) { + if ( + (scope === "tables" && (deletedTableIds.has(relationship.sourceTableId) || deletedTableIds.has(relationship.targetTableId))) + || (scope !== "tables" && !observed.has(`${relationship.sourceTableName}\u0000${relationship.constraintName}`)) + ) { + deletedRelationships.push({ + sourceTableName: relationship.sourceTableName, + constraintName: relationship.constraintName, + }); + } + } + deletedRelationships.sort((a, b) => `${a.sourceTableName}.${a.constraintName}`.localeCompare(`${b.sourceTableName}.${b.constraintName}`)); + } + return { deletedTables, deletedColumns, deletedRelationships }; + } + + async applySchemaSync( + databaseId: string, + expectedDatabaseVersion: number, + scope: CatalogSyncScope, + tableIds: readonly string[], + snapshot: ObservedSchemaSnapshot, + ): Promise { + return await this.db.transaction().execute(async (trx) => { + const database = await trx.selectFrom("workspaceDatabases").select("version") + .where("id", "=", databaseId).forUpdate().executeTakeFirst(); + if (!database || database.version !== expectedDatabaseVersion) return undefined; + const now = new Date().toISOString(); + let created = 0; + let updated = 0; + let deleted = 0; + + let tables = await trx.selectFrom("catalogTables").selectAll() + .where("databaseId", "=", databaseId).execute(); + if (scope === "tables" || scope === "all") { + const byName = new Map(tables.map((table) => [table.name, table])); + const observedNames = new Set(snapshot.tables.map((table) => table.name)); + const removed = tables.filter((table) => !observedNames.has(table.name)); + if (removed.length > 0) { + await trx.deleteFrom("catalogTables").where("id", "in", removed.map((table) => table.id)).execute(); + deleted += removed.length; + } + for (const observed of snapshot.tables) { + const current = byName.get(observed.name); + if (!current) { + await trx.insertInto("catalogTables").values({ + id: randomUUID(), databaseId, name: observed.name, sourceComment: observed.sourceComment, + description: null, generatedDescription: null, + lastSyncedDatabaseVersion: expectedDatabaseVersion, lastSyncedAt: now, + }).execute(); + created += 1; + } else { + const changed = current.sourceComment !== observed.sourceComment; + await trx.updateTable("catalogTables").set({ + sourceComment: observed.sourceComment, + lastSyncedDatabaseVersion: expectedDatabaseVersion, + lastSyncedAt: now, + ...(changed ? { version: sql`version + 1`, updatedAt: sql`now()` } : {}), + }).where("id", "=", current.id).execute(); + if (changed) updated += 1; + } + } + tables = await trx.selectFrom("catalogTables").selectAll().where("databaseId", "=", databaseId).execute(); + } + + if (scope === "columns" || scope === "all") { + const selected = new Set(tableIds); + const targetTables = scope === "columns" && selected.size > 0 + ? tables.filter((table) => selected.has(table.id)) + : tables; + for (const table of targetTables) { + const observedColumns = snapshot.columns.filter((column) => column.tableName === table.name); + const existing = await trx.selectFrom("catalogColumns").selectAll().where("tableId", "=", table.id).execute(); + const observedNames = new Set(observedColumns.map((column) => column.name)); + const removed = existing.filter((column) => !observedNames.has(column.name)); + if (removed.length > 0) { + await trx.deleteFrom("catalogColumns").where("id", "in", removed.map((column) => column.id)).execute(); + deleted += removed.length; + } + const byName = new Map(existing.map((column) => [column.name, column])); + for (const observed of observedColumns) { + const current = byName.get(observed.name); + if (!current) { + await trx.insertInto("catalogColumns").values({ + id: randomUUID(), tableId: table.id, name: observed.name, + ordinalPosition: observed.ordinalPosition, dataType: observed.dataType, + isNullable: observed.isNullable, defaultExpression: observed.defaultExpression, + primaryKeyPosition: observed.primaryKeyPosition, sourceComment: observed.sourceComment, + description: null, generatedDescription: null, + lastSyncedDatabaseVersion: expectedDatabaseVersion, lastSyncedAt: now, + }).execute(); + created += 1; + } else { + const changed = current.ordinalPosition !== observed.ordinalPosition + || current.dataType !== observed.dataType || current.isNullable !== observed.isNullable + || current.defaultExpression !== observed.defaultExpression + || current.primaryKeyPosition !== observed.primaryKeyPosition + || current.sourceComment !== observed.sourceComment; + await trx.updateTable("catalogColumns").set({ + ordinalPosition: observed.ordinalPosition, dataType: observed.dataType, + isNullable: observed.isNullable, defaultExpression: observed.defaultExpression, + primaryKeyPosition: observed.primaryKeyPosition, sourceComment: observed.sourceComment, + lastSyncedDatabaseVersion: expectedDatabaseVersion, lastSyncedAt: now, + ...(changed ? { version: sql`version + 1`, updatedAt: sql`now()` } : {}), + }).where("id", "=", current.id).execute(); + if (changed) updated += 1; + } + } + } + } + + if (scope === "relationships" || scope === "all") { + tables = await trx.selectFrom("catalogTables").selectAll().where("databaseId", "=", databaseId).execute(); + const tableByName = new Map(tables.map((table) => [table.name, table])); + const existing = await trx.selectFrom("catalogRelationships").selectAll() + .where("databaseId", "=", databaseId).execute(); + const tableNameById = new Map(tables.map((table) => [table.id, table.name])); + const observedKeys = new Set(snapshot.relationships.map((relationship) => + `${relationship.sourceTableName}\u0000${relationship.constraintName}`)); + const removed = existing.filter((relationship) => + !observedKeys.has(`${tableNameById.get(relationship.sourceTableId)}\u0000${relationship.constraintName}`)); + if (removed.length > 0) { + await trx.deleteFrom("catalogRelationships").where("id", "in", removed.map((relationship) => relationship.id)).execute(); + deleted += removed.length; + } + const byKey = new Map(existing.map((relationship) => + [`${tableNameById.get(relationship.sourceTableId)}\u0000${relationship.constraintName}`, relationship])); + for (const observed of snapshot.relationships) { + const sourceTable = tableByName.get(observed.sourceTableName); + const targetTable = tableByName.get(observed.targetTableName); + if (!sourceTable || !targetTable) throw new CatalogConnectorError("Relationship endpoint is not present in the catalog"); + const sourceColumns = await trx.selectFrom("catalogColumns").select(["id", "name"]) + .where("tableId", "=", sourceTable.id).execute(); + const targetColumns = await trx.selectFrom("catalogColumns").select(["id", "name"]) + .where("tableId", "=", targetTable.id).execute(); + const sourceByName = new Map(sourceColumns.map((column) => [column.name, column.id])); + const targetByName = new Map(targetColumns.map((column) => [column.name, column.id])); + const key = `${observed.sourceTableName}\u0000${observed.constraintName}`; + const current = byKey.get(key); + const relationshipId = current?.id ?? randomUUID(); + const desiredPairs = observed.columns.map((pair) => { + const sourceColumnId = sourceByName.get(pair.sourceColumnName); + const targetColumnId = targetByName.get(pair.targetColumnName); + if (!sourceColumnId || !targetColumnId) { + throw new CatalogConnectorError("Relationship column is not present in the catalog"); + } + return { relationshipId, position: pair.position, sourceColumnId, targetColumnId }; + }).sort((left, right) => left.position - right.position); + const currentPairs = current ? await trx.selectFrom("catalogRelationshipColumns") + .select(["position", "sourceColumnId", "targetColumnId"]) + .where("relationshipId", "=", current.id).orderBy("position").execute() : []; + const changed = Boolean(current && ( + current.targetTableId !== targetTable.id + || current.updateRule !== observed.updateRule + || current.deleteRule !== observed.deleteRule + || current.deferrable !== observed.deferrable + || current.initiallyDeferred !== observed.initiallyDeferred + || JSON.stringify(currentPairs) !== JSON.stringify(desiredPairs.map( + ({ position, sourceColumnId, targetColumnId }) => ({ position, sourceColumnId, targetColumnId }), + )) + )); + if (!current) { + await trx.insertInto("catalogRelationships").values({ + id: relationshipId, databaseId, constraintName: observed.constraintName, + sourceTableId: sourceTable.id, targetTableId: targetTable.id, + updateRule: observed.updateRule, deleteRule: observed.deleteRule, + deferrable: observed.deferrable, initiallyDeferred: observed.initiallyDeferred, + lastSyncedDatabaseVersion: expectedDatabaseVersion, lastSyncedAt: now, + }).execute(); + created += 1; + } else { + await trx.updateTable("catalogRelationships").set({ + targetTableId: targetTable.id, updateRule: observed.updateRule, + deleteRule: observed.deleteRule, deferrable: observed.deferrable, + initiallyDeferred: observed.initiallyDeferred, + lastSyncedDatabaseVersion: expectedDatabaseVersion, lastSyncedAt: now, + ...(changed ? { updatedAt: sql`now()` } : {}), + }).where("id", "=", current.id).execute(); + if (changed) { + await trx.deleteFrom("catalogRelationshipColumns").where("relationshipId", "=", current.id).execute(); + updated += 1; + } + } + if (!current || changed) { + for (const pair of desiredPairs) { + await trx.insertInto("catalogRelationshipColumns").values(pair).execute(); + } + } + } + } + + if (scope === "all") { + await trx.updateTable("workspaceDatabases").set({ + schemaSyncedVersion: expectedDatabaseVersion, + schemaSyncedAt: now, + }).where("id", "=", databaseId).execute(); + } + return { + tables: snapshot.tables.length, + columns: scope === "tables" ? undefined : snapshot.columns.length, + relationships: scope === "tables" || scope === "columns" ? undefined : snapshot.relationships.length, + created, updated, deleted, + }; + }); + } + + async createSyncRun( + databaseId: string, + scope: CatalogSyncScope, + tableIds: readonly string[], + requestedDatabaseVersion: number, + ): Promise { + try { + const row = await this.db.insertInto("catalogSyncRuns").values({ + id: randomUUID(), databaseId, scope, tableIds: JSON.stringify([...tableIds]), state: "queued", phase: "queued", + requestedDatabaseVersion, observedSnapshot: null, plannedDiff: null, confirmationToken: null, + counts: {}, errorCode: null, errorMessage: null, cancelRequested: false, + startedAt: null, finishedAt: null, heartbeatAt: null, + leaseOwner: null, leaseExpiresAt: null, + }).returningAll().executeTakeFirstOrThrow(); + return serializeSyncRun(row); + } catch (error: any) { + if (error?.code === "23505") throw new CatalogConflictError("A synchronization is already active for this database"); + throw error; + } + } + + async getSyncRun(runId: string): Promise { + const row = await this.db.selectFrom("catalogSyncRuns").selectAll().where("id", "=", runId).executeTakeFirst(); + return row ? serializeSyncRun(row) : undefined; + } + + async claimSyncRun(runId: string, workerId: string, leaseExpiresAt: string): Promise { + const now = new Date(); + const row = await this.db.updateTable("catalogSyncRuns").set({ + state: "running", + leaseOwner: workerId, + leaseExpiresAt, + heartbeatAt: now, + startedAt: sql`coalesce(started_at, now())`, + updatedAt: now, + }).where("id", "=", runId) + .where("state", "=", "queued") + .where((eb) => eb.or([ + eb("leaseOwner", "is", null), + eb("leaseExpiresAt", "<", now), + eb("leaseOwner", "=", workerId), + ])) + .returningAll().executeTakeFirst(); + return row ? serializeSyncRun(row) : undefined; + } + + async listSyncRuns(databaseId: string, limit = 50): Promise { + const rows = await this.db.selectFrom("catalogSyncRuns").selectAll() + .where("databaseId", "=", databaseId).orderBy("createdAt", "desc").limit(limit).execute(); + return rows.map(serializeSyncRun); + } + + async updateSyncRun(runId: string, update: CatalogSyncRunUpdate): Promise { + const values: any = { ...update, updatedAt: sql`now()` }; + const row = await this.db.updateTable("catalogSyncRuns").set(values) + .where("id", "=", runId).returningAll().executeTakeFirst(); + return row ? serializeSyncRun(row) : undefined; + } + + async requestSyncRunCancellation(runId: string): Promise { + return await this.updateSyncRun(runId, { cancelRequested: true }); + } + + async appendSyncEvent( + runId: string, + level: CatalogSyncEvent["level"], + eventType: string, + message: string, + data: Record = {}, + ): Promise { + return await this.db.transaction().execute(async (trx) => { + const current = await trx.selectFrom("catalogSyncEvents") + .select(sql`coalesce(max(sequence), 0)::int`.as("sequence")) + .where("runId", "=", runId).executeTakeFirst(); + const row = await trx.insertInto("catalogSyncEvents").values({ + runId, sequence: Number(current?.sequence ?? 0) + 1, level, eventType, message, data, + }).returningAll().executeTakeFirstOrThrow(); + return serializeSyncEvent(row); + }); + } + + async listSyncEvents(runId: string, afterSequence = 0): Promise { + const rows = await this.db.selectFrom("catalogSyncEvents").selectAll() + .where("runId", "=", runId).where("sequence", ">", afterSequence) + .orderBy("sequence").execute(); + return rows.map(serializeSyncEvent); + } + + async pruneSyncEvents(before: string): Promise { + await this.db.deleteFrom("catalogSyncEvents").where("createdAt", "<", new Date(before)).execute(); + } + + async interruptActiveSyncRuns(): Promise { + await this.db.updateTable("catalogSyncRuns").set({ + state: "interrupted", phase: "completed", errorCode: "worker_restarted", + errorMessage: "Synchronization was interrupted by a backend restart.", + finishedAt: sql`now()`, updatedAt: sql`now()`, + leaseOwner: null, leaseExpiresAt: null, + }).where("state", "in", ["queued", "running", "awaiting_confirmation", "applying"]).execute(); + } + + async reconcileTables( + databaseId: string, + expectedDatabaseVersion: number, + observed: readonly ObservedCatalogTable[], + confirmedDeletedNames: readonly string[], + ): Promise { + return await this.db.transaction().execute(async (trx) => { + const database = await trx.selectFrom("workspaceDatabases").select("version") + .where("id", "=", databaseId).forUpdate().executeTakeFirst(); + if (!database || database.version !== expectedDatabaseVersion) return undefined; + + const existing = await trx.selectFrom("catalogTables").selectAll() + .where("databaseId", "=", databaseId).orderBy("name").execute(); + const byName = new Map(existing.map((table) => [table.name, table])); + const observedNames = new Set(observed.map((table) => table.name)); + const deletedNames = existing + .filter((table) => !observedNames.has(table.name)) + .map((table) => table.name) + .sort(); + const confirmation = [...new Set(confirmedDeletedNames)].sort(); + if (deletedNames.length > 0 && JSON.stringify(deletedNames) !== JSON.stringify(confirmation)) { + return { kind: "confirmation_required", deletedNames }; + } + + let createdCount = 0; + let updatedCount = 0; + for (const table of observed) { + const current = byName.get(table.name); + if (!current) { + await trx.insertInto("catalogTables").values({ + id: randomUUID(), + databaseId, + name: table.name, + sourceComment: table.sourceComment, + description: null, + generatedDescription: null, + }).execute(); + createdCount += 1; + } else if (current.sourceComment !== table.sourceComment) { + await trx.updateTable("catalogTables").set({ + sourceComment: table.sourceComment, + version: sql`version + 1`, + updatedAt: sql`now()`, + }).where("id", "=", current.id).execute(); + updatedCount += 1; + } + } + if (deletedNames.length > 0) { + await trx.deleteFrom("catalogTables") + .where("databaseId", "=", databaseId) + .where("name", "in", deletedNames) + .execute(); + } + + const rows = await trx.selectFrom("catalogTables").selectAll() + .where("databaseId", "=", databaseId).orderBy("name").execute(); + return { + kind: "applied", + createdCount, + updatedCount, + deletedCount: deletedNames.length, + tables: rows.map(serializeTable), + }; + }); + } + + async available(): Promise { + try { + await sql`select 1`.execute(this.db); + return true; + } catch { + return false; + } + } + + async close(): Promise { await this.db.destroy(); } +} + +export class UnavailableCatalogRepository implements CatalogRepository { + private fail(): never { throw new CatalogUnavailableError("Database catalog is unavailable"); } + async list(): Promise { return this.fail(); } + async get(): Promise { return this.fail(); } + async getByWorkspace(): Promise { return this.fail(); } + async create(): Promise { return this.fail(); } + async update(): Promise { return this.fail(); } + async recordTest(): Promise { return this.fail(); } + async touch(): Promise { return this.fail(); } + async delete(): Promise { return this.fail(); } + async listTables(): Promise { return this.fail(); } + async getTable(): Promise { return this.fail(); } + async updateTableDescription(): Promise { return this.fail(); } + async updateTableMetadata(): Promise { return this.fail(); } + async listColumns(): Promise { return this.fail(); } + async getColumn(): Promise { return this.fail(); } + async updateColumnMetadata(): Promise { return this.fail(); } + async listRelationships(): Promise { return this.fail(); } + async planSchemaSync(): Promise { return this.fail(); } + async applySchemaSync(): Promise { return this.fail(); } + async createSyncRun(): Promise { return this.fail(); } + async getSyncRun(): Promise { return this.fail(); } + async claimSyncRun(): Promise { return this.fail(); } + async listSyncRuns(): Promise { return this.fail(); } + async updateSyncRun(): Promise { return this.fail(); } + async requestSyncRunCancellation(): Promise { return this.fail(); } + async appendSyncEvent(): Promise { return this.fail(); } + async listSyncEvents(): Promise { return this.fail(); } + async pruneSyncEvents(): Promise { return this.fail(); } + async interruptActiveSyncRuns(): Promise { return this.fail(); } + async reconcileTables(): Promise { return this.fail(); } + async available(): Promise { return false; } +} + +export interface CatalogConnectionConfig { + connectionString?: string; + host?: string; + port?: number; + database?: string; + user?: string; + passwordFile?: string; +} + +export function catalogPool(config: CatalogConnectionConfig, max = 5): Pool { + return new Pool(config.connectionString + ? { connectionString: config.connectionString, max, connectionTimeoutMillis: 3_000 } + : { + host: config.host, + port: config.port, + database: config.database, + user: config.user, + password: async () => (await readFile(config.passwordFile!, "utf8")).trim(), + max, + connectionTimeoutMillis: 3_000, + }); +} + +export function createCatalogRepository(config: CatalogConnectionConfig | undefined): CatalogRepository { + if (!config) return new UnavailableCatalogRepository(); + const pool = catalogPool(config); + const db = new Kysely({ + dialect: new PostgresDialect({ pool }), + plugins: [new CamelCasePlugin()], + }); + return new KyselyCatalogRepository(db); +} diff --git a/backend/src/catalog/schema-introspector.ts b/backend/src/catalog/schema-introspector.ts new file mode 100644 index 00000000..c6fbcf07 --- /dev/null +++ b/backend/src/catalog/schema-introspector.ts @@ -0,0 +1,498 @@ +import { readFile } from "node:fs/promises"; +import { z } from "zod"; +import type { WorkspaceSecretStore } from "../workspaces/secret-store.js"; +import type { CatalogPostgresAccess } from "./postgres-access.js"; +import { CATALOG_SECRET_IDS } from "./secrets.js"; +import { + CatalogConnectorError, + CatalogSchemaCapabilityUnavailableError, + type CatalogSyncPhase, + type ObservedCatalogColumn, + type ObservedCatalogRelationship, + type ObservedCatalogTable, + type ObservedSchemaSnapshot, + type WorkspaceDatabase, +} from "./types.js"; + +export type CatalogSchemaScanProgress = ( + phase: Extract, + counts?: { tables?: number; columns?: number; relationships?: number }, +) => Promise | void; + +export interface CatalogSchemaIntrospector { + scan( + database: WorkspaceDatabase, + signal: AbortSignal, + progress?: CatalogSchemaScanProgress, + ): Promise; +} + +const identifier = z.string().min(1).max(128); +const nullableText = z.string().nullable(); +const capability = z.enum(["available", "unavailable"]); +const restSnapshotSchema = z.object({ + schemaVersion: z.literal(1), + capabilities: z.object({ + tables: capability, + columns: capability, + relationships: capability, + }).strict(), + tables: z.array(z.object({ + name: identifier, + sourceComment: nullableText, + }).strict()), + columns: z.array(z.object({ + tableName: identifier, + name: identifier, + ordinalPosition: z.number().int().positive(), + dataType: z.string().min(1).max(2_000), + isNullable: z.boolean(), + defaultExpression: nullableText, + primaryKeyPosition: z.number().int().positive().nullable(), + sourceComment: nullableText, + }).strict()), + relationships: z.array(z.object({ + constraintName: identifier, + sourceTableName: identifier, + targetTableName: identifier, + updateRule: z.string().min(1).max(64), + deleteRule: z.string().min(1).max(64), + deferrable: z.boolean(), + initiallyDeferred: z.boolean(), + columns: z.array(z.object({ + position: z.number().int().positive(), + sourceColumnName: identifier, + targetColumnName: identifier, + }).strict()).min(1), + }).strict()), +}).strict(); + +function sqlString(value: string): string { + return `'${value.replaceAll("'", "''")}'`; +} + +function restSnapshotQuery(schemaName: string): string { + const schema = sqlString(schemaName); + return `WITH target_schema AS ( + SELECT oid + FROM pg_catalog.pg_namespace + WHERE nspname = ${schema} + ), + observed_tables AS ( + SELECT c.oid, + c.relname AS name, + d.description AS source_comment + FROM pg_catalog.pg_class c + JOIN target_schema n ON n.oid = c.relnamespace + LEFT JOIN pg_catalog.pg_description d ON d.objoid = c.oid AND d.objsubid = 0 + WHERE c.relkind IN ('r', 'p') + ), + primary_key_columns AS ( + SELECT i.indrelid AS table_oid, + key.attnum, + key.ordinality::integer AS position + FROM pg_catalog.pg_index i + CROSS JOIN LATERAL unnest(i.indkey) WITH ORDINALITY AS key(attnum, ordinality) + WHERE i.indisprimary + ), + observed_columns AS ( + SELECT table_info.name AS table_name, + a.attname AS name, + a.attnum::integer AS ordinal_position, + pg_catalog.format_type(a.atttypid, a.atttypmod) AS data_type, + NOT a.attnotnull AS is_nullable, + pg_catalog.pg_get_expr(ad.adbin, ad.adrelid) AS default_expression, + pk.position AS primary_key_position, + d.description AS source_comment + FROM observed_tables table_info + JOIN pg_catalog.pg_attribute a ON a.attrelid = table_info.oid + LEFT JOIN pg_catalog.pg_attrdef ad ON ad.adrelid = table_info.oid AND ad.adnum = a.attnum + LEFT JOIN pg_catalog.pg_description d ON d.objoid = table_info.oid AND d.objsubid = a.attnum + LEFT JOIN primary_key_columns pk ON pk.table_oid = table_info.oid AND pk.attnum = a.attnum + WHERE a.attnum > 0 + AND NOT a.attisdropped + ), + relationship_pairs AS ( + SELECT con.oid AS constraint_oid, + con.conname AS constraint_name, + source_table.relname AS source_table_name, + target_table.relname AS target_table_name, + CASE con.confupdtype + WHEN 'a' THEN 'NO ACTION' + WHEN 'r' THEN 'RESTRICT' + WHEN 'c' THEN 'CASCADE' + WHEN 'n' THEN 'SET NULL' + WHEN 'd' THEN 'SET DEFAULT' + END AS update_rule, + CASE con.confdeltype + WHEN 'a' THEN 'NO ACTION' + WHEN 'r' THEN 'RESTRICT' + WHEN 'c' THEN 'CASCADE' + WHEN 'n' THEN 'SET NULL' + WHEN 'd' THEN 'SET DEFAULT' + END AS delete_rule, + con.condeferrable AS is_deferrable, + con.condeferred AS initially_deferred, + source_key.ordinality::integer AS position, + source_column.attname AS source_column_name, + target_column.attname AS target_column_name + FROM pg_catalog.pg_constraint con + JOIN pg_catalog.pg_class source_table ON source_table.oid = con.conrelid + JOIN target_schema source_namespace ON source_namespace.oid = source_table.relnamespace + JOIN pg_catalog.pg_class target_table ON target_table.oid = con.confrelid + JOIN target_schema target_namespace ON target_namespace.oid = target_table.relnamespace + JOIN LATERAL unnest(con.conkey) WITH ORDINALITY AS source_key(attnum, ordinality) ON true + JOIN LATERAL unnest(con.confkey) WITH ORDINALITY AS target_key(attnum, ordinality) + ON target_key.ordinality = source_key.ordinality + JOIN pg_catalog.pg_attribute source_column + ON source_column.attrelid = source_table.oid AND source_column.attnum = source_key.attnum + JOIN pg_catalog.pg_attribute target_column + ON target_column.attrelid = target_table.oid AND target_column.attnum = target_key.attnum + WHERE con.contype = 'f' + ), + observed_relationships AS ( + SELECT constraint_oid, + constraint_name, + source_table_name, + target_table_name, + update_rule, + delete_rule, + is_deferrable, + initially_deferred, + pg_catalog.jsonb_agg( + pg_catalog.jsonb_build_object( + 'position', position, + 'sourceColumnName', source_column_name, + 'targetColumnName', target_column_name + ) ORDER BY position + ) AS columns + FROM relationship_pairs + GROUP BY constraint_oid, constraint_name, source_table_name, target_table_name, + update_rule, delete_rule, is_deferrable, initially_deferred + ) + SELECT 1 AS "schemaVersion", + pg_catalog.jsonb_build_object( + 'tables', 'available', + 'columns', 'available', + 'relationships', 'available' + ) AS capabilities, + COALESCE(( + SELECT pg_catalog.jsonb_agg( + pg_catalog.jsonb_build_object('name', name, 'sourceComment', source_comment) + ORDER BY name + ) + FROM observed_tables + ), '[]'::jsonb) AS tables, + COALESCE(( + SELECT pg_catalog.jsonb_agg( + pg_catalog.jsonb_build_object( + 'tableName', table_name, + 'name', name, + 'ordinalPosition', ordinal_position, + 'dataType', data_type, + 'isNullable', is_nullable, + 'defaultExpression', default_expression, + 'primaryKeyPosition', primary_key_position, + 'sourceComment', source_comment + ) ORDER BY table_name, ordinal_position + ) + FROM observed_columns + ), '[]'::jsonb) AS columns, + COALESCE(( + SELECT pg_catalog.jsonb_agg( + pg_catalog.jsonb_build_object( + 'constraintName', constraint_name, + 'sourceTableName', source_table_name, + 'targetTableName', target_table_name, + 'updateRule', update_rule, + 'deleteRule', delete_rule, + 'deferrable', is_deferrable, + 'initiallyDeferred', initially_deferred, + 'columns', columns + ) ORDER BY source_table_name, constraint_name + ) + FROM observed_relationships + ), '[]'::jsonb) AS relationships + FROM target_schema`; +} + +function required(value: string | undefined): string { + if (!value) throw new CatalogConnectorError("Database binding is incomplete"); + return value; +} + +function textOrNull(value: unknown): string | null { + return typeof value === "string" && value.length > 0 ? value : null; +} + +function actionRule(value: unknown): string { + const rules: Record = { + a: "NO ACTION", + r: "RESTRICT", + c: "CASCADE", + n: "SET NULL", + d: "SET DEFAULT", + }; + const rule = rules[String(value)]; + if (!rule) throw new CatalogConnectorError("Schema introspection returned an unknown relationship action"); + return rule; +} + +function normalized(snapshot: ObservedSchemaSnapshot): ObservedSchemaSnapshot { + const tables = new Map(); + for (const table of snapshot.tables) { + if (tables.has(table.name)) throw new CatalogConnectorError("Schema introspection returned duplicate tables"); + tables.set(table.name, table); + } + const columns = new Map(); + for (const column of snapshot.columns) { + const key = `${column.tableName}\u0000${column.name}`; + if (columns.has(key)) throw new CatalogConnectorError("Schema introspection returned duplicate columns"); + columns.set(key, column); + } + const relationships = new Map(); + for (const relationship of snapshot.relationships) { + const key = `${relationship.sourceTableName}\u0000${relationship.constraintName}`; + if (relationships.has(key)) throw new CatalogConnectorError("Schema introspection returned duplicate relationships"); + relationships.set(key, { + ...relationship, + columns: [...relationship.columns].sort((a, b) => a.position - b.position), + }); + } + return { + schemaVersion: 1, + capabilities: snapshot.capabilities, + tables: [...tables.values()].sort((a, b) => a.name.localeCompare(b.name)), + columns: [...columns.values()].sort((a, b) => ( + a.tableName.localeCompare(b.tableName) || a.ordinalPosition - b.ordinalPosition + )), + relationships: [...relationships.values()].sort((a, b) => ( + a.sourceTableName.localeCompare(b.sourceTableName) || a.constraintName.localeCompare(b.constraintName) + )), + }; +} + +export class ConcreteCatalogSchemaIntrospector implements CatalogSchemaIntrospector { + constructor( + private readonly postgres: CatalogPostgresAccess, + private readonly secretStore: WorkspaceSecretStore, + ) {} + + async scan( + database: WorkspaceDatabase, + signal: AbortSignal, + progress?: CatalogSchemaScanProgress, + ): Promise { + return database.binding.transport === "rest_api" + ? await this.scanRest(database, signal, progress) + : await this.scanPostgres(database, signal, progress); + } + + private async scanPostgres( + database: WorkspaceDatabase, + signal: AbortSignal, + progress?: CatalogSchemaScanProgress, + ): Promise { + await progress?.("connecting"); + const client = await this.postgres.connect(database, signal); + try { + const schema = await client.query( + "SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname = $1) AS present", + [database.schema], + ); + if (schema.rows[0]?.present !== true) throw new CatalogConnectorError("Database schema is unavailable"); + + await progress?.("scanning_tables"); + const tableResult = await client.query( + `SELECT c.relname AS name, d.description AS source_comment + FROM pg_catalog.pg_class c + JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace + LEFT JOIN pg_catalog.pg_description d ON d.objoid = c.oid AND d.objsubid = 0 + WHERE c.relkind IN ('r', 'p') AND n.nspname = $1 + ORDER BY c.relname`, + [database.schema], + ); + const tables: ObservedCatalogTable[] = tableResult.rows.map((row) => ({ + name: String(row.name), + sourceComment: textOrNull(row.source_comment), + })); + await progress?.("scanning_tables", { tables: tables.length }); + + await progress?.("scanning_columns", { tables: tables.length }); + const columnResult = await client.query( + `SELECT c.relname AS table_name, + a.attname AS name, + a.attnum::integer AS ordinal_position, + pg_catalog.format_type(a.atttypid, a.atttypmod) AS data_type, + NOT a.attnotnull AS is_nullable, + pg_catalog.pg_get_expr(ad.adbin, ad.adrelid) AS default_expression, + pk.position AS primary_key_position, + d.description AS source_comment + FROM pg_catalog.pg_class c + JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace + JOIN pg_catalog.pg_attribute a ON a.attrelid = c.oid + LEFT JOIN pg_catalog.pg_attrdef ad ON ad.adrelid = c.oid AND ad.adnum = a.attnum + LEFT JOIN pg_catalog.pg_description d ON d.objoid = c.oid AND d.objsubid = a.attnum + LEFT JOIN LATERAL ( + SELECT key.ordinality::integer AS position + FROM pg_catalog.pg_index i + CROSS JOIN LATERAL unnest(i.indkey) WITH ORDINALITY AS key(attnum, ordinality) + WHERE i.indrelid = c.oid AND i.indisprimary AND key.attnum = a.attnum + LIMIT 1 + ) pk ON true + WHERE c.relkind IN ('r', 'p') + AND n.nspname = $1 + AND a.attnum > 0 + AND NOT a.attisdropped + ORDER BY c.relname, a.attnum`, + [database.schema], + ); + const columns: ObservedCatalogColumn[] = columnResult.rows.map((row) => ({ + tableName: String(row.table_name), + name: String(row.name), + ordinalPosition: Number(row.ordinal_position), + dataType: String(row.data_type), + isNullable: row.is_nullable === true, + defaultExpression: textOrNull(row.default_expression), + primaryKeyPosition: row.primary_key_position === null || row.primary_key_position === undefined + ? null + : Number(row.primary_key_position), + sourceComment: textOrNull(row.source_comment), + })); + await progress?.("scanning_columns", { tables: tables.length, columns: columns.length }); + + await progress?.("scanning_relationships", { tables: tables.length, columns: columns.length }); + const relationshipResult = await client.query( + `SELECT con.conname AS constraint_name, + source_table.relname AS source_table_name, + target_table.relname AS target_table_name, + con.confupdtype AS update_action, + con.confdeltype AS delete_action, + con.condeferrable AS deferrable, + con.condeferred AS initially_deferred, + source_key.ordinality::integer AS position, + source_column.attname AS source_column_name, + target_column.attname AS target_column_name + FROM pg_catalog.pg_constraint con + JOIN pg_catalog.pg_class source_table ON source_table.oid = con.conrelid + JOIN pg_catalog.pg_namespace source_namespace ON source_namespace.oid = source_table.relnamespace + JOIN pg_catalog.pg_class target_table ON target_table.oid = con.confrelid + JOIN pg_catalog.pg_namespace target_namespace ON target_namespace.oid = target_table.relnamespace + JOIN LATERAL unnest(con.conkey) WITH ORDINALITY AS source_key(attnum, ordinality) ON true + JOIN LATERAL unnest(con.confkey) WITH ORDINALITY AS target_key(attnum, ordinality) + ON target_key.ordinality = source_key.ordinality + JOIN pg_catalog.pg_attribute source_column + ON source_column.attrelid = source_table.oid AND source_column.attnum = source_key.attnum + JOIN pg_catalog.pg_attribute target_column + ON target_column.attrelid = target_table.oid AND target_column.attnum = target_key.attnum + WHERE con.contype = 'f' + AND source_namespace.nspname = $1 + AND target_namespace.nspname = $1 + ORDER BY source_table.relname, con.conname, source_key.ordinality`, + [database.schema], + ); + const relationshipMap = new Map(); + for (const row of relationshipResult.rows) { + const sourceTableName = String(row.source_table_name); + const constraintName = String(row.constraint_name); + const key = `${sourceTableName}\u0000${constraintName}`; + const current = relationshipMap.get(key) ?? { + constraintName, + sourceTableName, + targetTableName: String(row.target_table_name), + updateRule: actionRule(row.update_action), + deleteRule: actionRule(row.delete_action), + deferrable: row.deferrable === true, + initiallyDeferred: row.initially_deferred === true, + columns: [], + }; + current.columns.push({ + position: Number(row.position), + sourceColumnName: String(row.source_column_name), + targetColumnName: String(row.target_column_name), + }); + relationshipMap.set(key, current); + } + const relationships = [...relationshipMap.values()]; + await progress?.("scanning_relationships", { + tables: tables.length, + columns: columns.length, + relationships: relationships.length, + }); + return normalized({ + schemaVersion: 1, + capabilities: { tables: "available", columns: "available", relationships: "available" }, + tables, + columns, + relationships, + }); + } finally { + await client.end(); + } + } + + private async scanRest( + database: WorkspaceDatabase, + signal: AbortSignal, + progress?: CatalogSchemaScanProgress, + ): Promise { + await progress?.("connecting"); + const auth = database.binding.restAuth ?? "bearer"; + const materialized = this.secretStore.materialize( + database.workspaceId, + auth === "none" ? [] : [CATALOG_SECRET_IDS.apiKey], + ); + try { + const headers: Record = { "content-type": "application/json" }; + if (auth !== "none") { + const credentialFile = materialized.files.get(CATALOG_SECRET_IDS.apiKey); + if (!credentialFile) throw new CatalogConnectorError("REST API key is not configured"); + const credential = (await readFile(credentialFile, "utf8")).trim(); + if (auth === "bearer") headers.authorization = `Bearer ${credential}`; + else headers["x-api-key"] = credential; + } + const baseUrl = required(database.binding.baseUrl).replace(/\/+$/, ""); + const response = await fetch(`${baseUrl}/rpc/schema_snapshot`, { + method: "POST", + headers, + body: JSON.stringify({ schema_name: database.schema }), + signal, + }); + let body: unknown; + if (response.ok) { + body = await response.json(); + } else if (response.status === 404) { + const fallback = await fetch(`${baseUrl}/rpc/run_query`, { + method: "POST", + headers, + body: JSON.stringify({ query_text: restSnapshotQuery(database.schema) }), + signal, + }); + if (!fallback.ok) throw new CatalogSchemaCapabilityUnavailableError("schema_snapshot"); + const rows: unknown = await fallback.json(); + if (!Array.isArray(rows) || rows.length !== 1) { + throw new CatalogConnectorError("REST schema snapshot fallback is invalid"); + } + body = rows[0]; + } else { + throw new CatalogSchemaCapabilityUnavailableError("schema_snapshot"); + } + const parsed = restSnapshotSchema.safeParse(body); + if (!parsed.success) throw new CatalogConnectorError("REST schema snapshot is invalid"); + const snapshot = normalized(parsed.data); + await progress?.("scanning_tables", { tables: snapshot.tables.length }); + await progress?.("scanning_columns", { tables: snapshot.tables.length, columns: snapshot.columns.length }); + await progress?.("scanning_relationships", { + tables: snapshot.tables.length, + columns: snapshot.columns.length, + relationships: snapshot.relationships.length, + }); + return snapshot; + } catch (error) { + if (error instanceof CatalogConnectorError) throw error; + throw new CatalogConnectorError("REST schema introspection failed"); + } finally { + materialized.release(); + } + } +} diff --git a/backend/src/catalog/secrets.ts b/backend/src/catalog/secrets.ts new file mode 100644 index 00000000..86cc4d73 --- /dev/null +++ b/backend/src/catalog/secrets.ts @@ -0,0 +1,10 @@ +export const CATALOG_SECRET_IDS = { + password: "catalog.dwh.password", + apiKey: "catalog.dwh.api_key", + sshPrivateKey: "catalog.dwh.ssh_private_key", + sshPrivateKeyPassphrase: "catalog.dwh.ssh_private_key_passphrase", + sshKnownHosts: "catalog.dwh.ssh_known_hosts", + tlsCa: "catalog.dwh.tls_ca", +} as const; + +export type CatalogSecretName = keyof typeof CATALOG_SECRET_IDS; diff --git a/backend/src/catalog/service.ts b/backend/src/catalog/service.ts new file mode 100644 index 00000000..454d3a0f --- /dev/null +++ b/backend/src/catalog/service.ts @@ -0,0 +1,228 @@ +import { buildInstallationContract } from "../workspaces/contracts.js"; +import { resolveBinding } from "../workspaces/bindings.js"; +import type { WorkspaceRegistry } from "../workspaces/registry.js"; +import type { WorkspaceDescriptor } from "../workspaces/schema.js"; +import type { WorkspaceSecretStore } from "../workspaces/secret-store.js"; +import { createConcreteDiagnosticAdapters } from "../workspaces/diagnostics.js"; +import { CatalogOperationCoordinator } from "./operation-coordinator.js"; +import { + ConcreteCatalogPostgresAccess, + type CatalogPostgresAccess, +} from "./postgres-access.js"; +import type { + CatalogRepository, + DatabaseBinding, + DatabaseConfigurationInput, + DatabaseTestResult, + WorkspaceDatabase, +} from "./types.js"; +import { CATALOG_SECRET_IDS, type CatalogSecretName } from "./secrets.js"; + +export { CATALOG_SECRET_IDS, type CatalogSecretName } from "./secrets.js"; + +export interface CatalogListItem extends Omit { + id?: string; + workspaceName: string; + workspaceDescription?: string; + workspaceAvailable: boolean; + configured: boolean; + secrets: Record; +} + +function bindingValue(workspace: WorkspaceDescriptor, values: Record, suffix: string) { + const variable = buildInstallationContract(workspace).variables.find((entry) => ( + entry.role === "DWH" && entry.suffix === suffix + )); + return variable ? values[variable.name] : undefined; +} + +function numeric(value: string | undefined): number | undefined { + if (!value) return undefined; + const parsed = Number(value); + return Number.isInteger(parsed) && parsed >= 1 && parsed <= 65_535 ? parsed : undefined; +} + +function yamlBinding(workspace: WorkspaceDescriptor, secretRoots: readonly string[]): DatabaseBinding { + const effective = resolveBinding(workspace, "DWH", process.env, secretRoots); + const value = (suffix: string) => bindingValue(workspace, effective.values, suffix); + return { + transport: effective.transport, + host: value("HOST"), + port: numeric(value("PORT")) ?? workspace.dwh.port, + username: value("USER"), + baseUrl: value("BASE_URL"), + restPath: workspace.diagnostics?.dwh_rest?.path ?? "/health", + restAuth: workspace.diagnostics?.dwh_rest?.auth ?? "bearer", + tlsServername: value("TLS_SERVERNAME"), + sshHost: value("SSH_HOST"), + sshPort: numeric(value("SSH_PORT")), + sshUsername: value("SSH_USER"), + sshTargetHost: value("SSH_TARGET_HOST"), + sshTargetPort: numeric(value("SSH_TARGET_PORT")), + }; +} + +function secretState(store: WorkspaceSecretStore, workspaceId: string): Record { + return Object.fromEntries(Object.entries(CATALOG_SECRET_IDS).map(([name, id]) => ( + [name, store.has(workspaceId, id)] + ))) as Record; +} + +export class CatalogService { + private readonly adapters = createConcreteDiagnosticAdapters(); + + constructor( + private readonly repository: CatalogRepository, + private readonly registry: WorkspaceRegistry, + private readonly secretStore: WorkspaceSecretStore, + private readonly secretRoots: readonly string[], + private readonly diagnosticTimeoutMs: number, + private readonly postgres: CatalogPostgresAccess = new ConcreteCatalogPostgresAccess(secretStore, { + connectTimeoutMs: diagnosticTimeoutMs, + }), + private readonly operations: CatalogOperationCoordinator = new CatalogOperationCoordinator(), + ) {} + + async list(): Promise { + const [workspaces, configured] = await Promise.all([ + this.registry.listCatalog(), + this.repository.list(), + ]); + const byWorkspace = new Map(configured.map((database) => [database.workspaceId, database])); + const active = await Promise.all(workspaces.map(async (entry) => { + const database = byWorkspace.get(entry.id); + const { workspace } = await this.registry.read(entry.id); + const base = database ?? { + workspaceId: entry.id, + engine: "postgres" as const, + databaseName: workspace.dwh.database, + schema: workspace.dwh.schema, + version: 0, + createdAt: "", + updatedAt: "", + binding: yamlBinding(workspace, this.secretRoots), + connectionStatus: "untested" as const, + }; + return { + ...base, + workspaceName: entry.name, + workspaceDescription: entry.description, + workspaceAvailable: true, + configured: database !== undefined, + secrets: secretState(this.secretStore, entry.id), + }; + })); + const known = new Set(workspaces.map((entry) => entry.id)); + const orphaned: CatalogListItem[] = configured + .filter((database) => !known.has(database.workspaceId)) + .map((database) => ({ + ...database, + workspaceName: database.workspaceId, + workspaceDescription: "Workspace is no longer present in the repository catalog.", + workspaceAvailable: false, + configured: true, + secrets: secretState(this.secretStore, database.workspaceId), + })); + return [...active, ...orphaned]; + } + + async ensureWorkspace(workspaceId: string): Promise { + const { workspace } = await this.registry.read(workspaceId); + return workspace; + } + + async normalizeInput(input: DatabaseConfigurationInput): Promise { + const workspace = await this.ensureWorkspace(input.workspaceId); + if (input.binding.transport !== "rest_api") return input; + return { + ...input, + binding: { + ...input.binding, + restPath: workspace.diagnostics?.dwh_rest?.path ?? "/health", + }, + }; + } + + configuredSecrets(workspaceId: string): Record { + return secretState(this.secretStore, workspaceId); + } + + replaceSecrets(workspaceId: string, values: Partial>): void { + const encoded: Record = {}; + for (const [name, value] of Object.entries(values) as Array<[CatalogSecretName, string | undefined]>) { + if (value !== undefined && value.length > 0) encoded[CATALOG_SECRET_IDS[name]] = value; + } + if (Object.keys(encoded).length > 0) this.secretStore.putMany(workspaceId, encoded); + } + + forgetSecrets(workspaceId: string): void { + for (const id of Object.values(CATALOG_SECRET_IDS)) this.secretStore.forget(workspaceId, id); + } + + async test(database: WorkspaceDatabase): Promise { + return await this.operations.run(database.id, async () => { + const testedAt = new Date().toISOString(); + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), this.diagnosticTimeoutMs); + const required = database.binding.transport === "rest_api" + ? database.binding.restAuth === "none" ? [] : [CATALOG_SECRET_IDS.apiKey] + : []; + const materialized = this.secretStore.materialize(database.workspaceId, required); + try { + if (database.binding.transport !== "rest_api") { + const client = await this.postgres.connect(database, controller.signal); + try { + const result = await client.query( + `SELECT current_database() AS database, + CASE WHEN pg_catalog.has_schema_privilege( + current_user, + (SELECT oid FROM pg_catalog.pg_namespace WHERE nspname = $1), + 'USAGE' + ) THEN $1 ELSE NULL END AS schema`, + [database.schema], + ); + const row = result.rows[0]; + if (row?.database !== database.databaseName || row.schema !== database.schema) { + throw new Error("Database identity mismatch"); + } + } finally { + await client.end(); + } + } else { + const credentialId = CATALOG_SECRET_IDS.apiKey; + await this.adapters.probeConnector({ + role: "dwh", + transport: database.binding.transport, + baseUrl: database.binding.baseUrl, + credentialFile: materialized.files.get(credentialId), + resource: { database: database.databaseName, schema: database.schema }, + timeoutMs: this.diagnosticTimeoutMs, + signal: controller.signal, + diagnostic: { + method: "GET" as const, + path: database.binding.restPath ?? "/health", + auth: database.binding.restAuth ?? "bearer", + }, + }); + } + return { + connectionStatus: "reachable", + testedVersion: database.version, + lastTestedAt: testedAt, + }; + } catch { + return { + connectionStatus: "failed", + testedVersion: database.version, + lastTestedAt: testedAt, + errorCode: "connector_unavailable", + errorMessage: "The database connector could not be reached or authenticated.", + }; + } finally { + clearTimeout(timer); + controller.abort(); + materialized.release(); + } + }); + } +} diff --git a/backend/src/catalog/sync-worker.ts b/backend/src/catalog/sync-worker.ts new file mode 100644 index 00000000..8f837b63 --- /dev/null +++ b/backend/src/catalog/sync-worker.ts @@ -0,0 +1,311 @@ +import { randomUUID } from "node:crypto"; +import type { CatalogOperationCoordinator } from "./operation-coordinator.js"; +import type { CatalogSchemaIntrospector, CatalogSchemaScanProgress } from "./schema-introspector.js"; +import { + CatalogConflictError, + CatalogConnectorError, + CatalogSchemaCapabilityUnavailableError, + type CatalogRepository, + type CatalogSchemaDiff, + type CatalogSyncCounts, + type CatalogSyncRun, + type CatalogSyncScope, + type ObservedSchemaSnapshot, + type WorkspaceDatabase, +} from "./types.js"; + +class SyncCancelledError extends Error {} + +const TERMINAL_STATES = new Set([ + "succeeded", "failed", "cancelled", "interrupted", +]); + +function destructive(diff: CatalogSchemaDiff): boolean { + return diff.deletedTables.length > 0 + || diff.deletedColumns.length > 0 + || diff.deletedRelationships.length > 0; +} + +function fingerprint(snapshot: ObservedSchemaSnapshot): string { + return JSON.stringify(snapshot); +} + +function safeFailure(error: unknown): { code: string; message: string } { + if (error instanceof CatalogSchemaCapabilityUnavailableError) { + const label = error.capability === "schema_snapshot" ? "schema snapshot" : error.capability; + return { + code: "schema_capability_unavailable", + message: `This database binding does not provide the ${label} capability.`, + }; + } + if (error instanceof CatalogConnectorError) { + return { code: "schema_introspection_failed", message: "The database schema could not be read safely." }; + } + return { code: "schema_sync_failed", message: "Schema synchronization failed." }; +} + +export class CatalogSyncWorker { + private readonly workerId = randomUUID(); + private readonly controllers = new Map(); + private readonly reservations = new Map void>(); + private stopping = false; + + constructor( + private readonly repository: CatalogRepository, + private readonly introspector: CatalogSchemaIntrospector, + private readonly operations: CatalogOperationCoordinator, + private readonly timeoutMs: number, + ) {} + + async initialize(): Promise { + if (!(await this.repository.available())) return; + await this.repository.interruptActiveSyncRuns(); + await this.repository.pruneSyncEvents(new Date(Date.now() - 30 * 24 * 60 * 60 * 1_000).toISOString()); + } + + async start(database: WorkspaceDatabase, scope: CatalogSyncScope, tableIds: readonly string[]): Promise { + this.assertReady(database); + const uniqueTableIds = [...new Set(tableIds)]; + if (scope === "columns") { + const tables = await Promise.all(uniqueTableIds.map((tableId) => this.repository.getTable(database.id, tableId))); + if (tables.some((table) => !table)) throw new CatalogConflictError("One or more selected tables no longer exist"); + } + if (scope !== "columns" && uniqueTableIds.length > 0) { + throw new CatalogConflictError("Table selection is only valid for a column synchronization"); + } + const release = this.operations.reserve(database.id); + try { + const run = await this.repository.createSyncRun(database.id, scope, uniqueTableIds, database.version); + this.reservations.set(run.id, release); + await this.repository.appendSyncEvent(run.id, "info", "queued", "Synchronization queued.", { scope }); + this.launch(run.id); + return run; + } catch (error) { + release(); + throw error; + } + } + + async confirm(runId: string, confirmationToken: string): Promise { + const run = await this.repository.getSyncRun(runId); + if (!run) return undefined; + if (run.state !== "awaiting_confirmation" || !run.observedSnapshot || run.confirmationToken !== confirmationToken) { + throw new CatalogConflictError("Synchronization confirmation is no longer valid"); + } + await this.repository.appendSyncEvent(run.id, "info", "confirmation_received", "Destructive changes were confirmed."); + const queued = await this.repository.updateSyncRun(run.id, { + state: "queued", + phase: "queued", + confirmationToken: null, + leaseOwner: null, + leaseExpiresAt: null, + }); + this.launch(run.id, fingerprint(run.observedSnapshot)); + return queued; + } + + async cancel(runId: string): Promise { + const run = await this.repository.getSyncRun(runId); + if (!run) return undefined; + if (run.state === "applying" || TERMINAL_STATES.has(run.state)) return run; + await this.repository.requestSyncRunCancellation(runId); + this.controllers.get(runId)?.abort(); + if (run.state === "queued" || run.state === "awaiting_confirmation") { + const cancelled = await this.repository.updateSyncRun(runId, { + state: "cancelled", + phase: "completed", + finishedAt: new Date().toISOString(), + observedSnapshot: null, + plannedDiff: null, + confirmationToken: null, + leaseOwner: null, + leaseExpiresAt: null, + }); + await this.repository.appendSyncEvent(runId, "warning", "cancelled", "Synchronization cancelled."); + this.release(runId); + return cancelled; + } + return await this.repository.getSyncRun(runId); + } + + async retry(runId: string): Promise { + const previous = await this.repository.getSyncRun(runId); + if (!previous) return undefined; + if (!TERMINAL_STATES.has(previous.state)) { + throw new CatalogConflictError("Only a finished synchronization can be retried"); + } + const database = await this.repository.get(previous.databaseId); + if (!database) return undefined; + return await this.start(database, previous.scope, previous.tableIds); + } + + async stop(): Promise { + this.stopping = true; + for (const controller of this.controllers.values()) controller.abort(); + if (await this.repository.available()) await this.repository.interruptActiveSyncRuns(); + for (const runId of [...this.reservations.keys()]) this.release(runId); + } + + private launch(runId: string, confirmedFingerprint?: string): void { + queueMicrotask(() => { + void this.execute(runId, confirmedFingerprint).catch(() => undefined); + }); + } + + private async execute(runId: string, confirmedFingerprint?: string): Promise { + if (this.stopping) return; + const leaseExpiresAt = new Date(Date.now() + 20_000).toISOString(); + const claimed = await this.repository.claimSyncRun(runId, this.workerId, leaseExpiresAt); + if (!claimed) return; + const controller = new AbortController(); + this.controllers.set(runId, controller); + let timedOut = false; + const timeout = setTimeout(() => { + timedOut = true; + controller.abort(); + }, this.timeoutMs); + const heartbeat = setInterval(() => { + void this.repository.updateSyncRun(runId, { + heartbeatAt: new Date().toISOString(), + leaseExpiresAt: new Date(Date.now() + 20_000).toISOString(), + }); + }, 5_000); + + try { + await this.repository.appendSyncEvent(runId, "info", "started", "Synchronization started."); + const database = await this.repository.get(claimed.databaseId); + if (!database || database.version !== claimed.requestedDatabaseVersion) { + throw new CatalogConflictError("Database binding changed before synchronization started"); + } + this.assertReady(database); + const progress: CatalogSchemaScanProgress = async (phase, counts) => { + await this.checkCancelled(runId); + await this.repository.updateSyncRun(runId, { + phase, + heartbeatAt: new Date().toISOString(), + ...(counts ? { counts } : {}), + }); + await this.repository.appendSyncEvent(runId, "info", phase, this.phaseMessage(phase), counts ?? {}); + }; + const snapshot = await this.introspector.scan(database, controller.signal, progress); + await this.checkCancelled(runId); + this.assertCapability(claimed.scope, snapshot); + const counts: CatalogSyncCounts = { + tables: snapshot.tables.length, + columns: snapshot.columns.length, + relationships: snapshot.relationships.length, + }; + await this.repository.updateSyncRun(runId, { phase: "planning", counts, observedSnapshot: snapshot }); + await this.repository.appendSyncEvent(runId, "info", "planning", "Schema changes are being planned.", { ...counts }); + const diff = await this.repository.planSchemaSync(claimed.databaseId, claimed.scope, claimed.tableIds, snapshot); + await this.checkCancelled(runId); + if (destructive(diff) && fingerprint(snapshot) !== confirmedFingerprint) { + const token = randomUUID(); + const waiting = await this.repository.updateSyncRun(runId, { + state: "awaiting_confirmation", + phase: "awaiting_confirmation", + observedSnapshot: snapshot, + plannedDiff: diff, + confirmationToken: token, + counts, + leaseOwner: null, + leaseExpiresAt: null, + }); + await this.repository.appendSyncEvent(runId, "warning", "confirmation_required", "Confirmation is required before removing catalog objects.", { + deletedTables: diff.deletedTables.length, + deletedColumns: diff.deletedColumns.length, + deletedRelationships: diff.deletedRelationships.length, + }); + if (!waiting) throw new Error("Synchronization run disappeared"); + return; + } + + await this.repository.updateSyncRun(runId, { state: "applying", phase: "applying", plannedDiff: diff }); + await this.repository.appendSyncEvent(runId, "info", "applying", "Catalog changes are being applied atomically."); + this.controllers.delete(runId); + const applied = await this.repository.applySchemaSync( + claimed.databaseId, + claimed.requestedDatabaseVersion, + claimed.scope, + claimed.tableIds, + snapshot, + ); + if (!applied) throw new CatalogConflictError("Database binding changed before schema changes were applied"); + await this.repository.updateSyncRun(runId, { + state: "succeeded", + phase: "completed", + counts: applied, + finishedAt: new Date().toISOString(), + observedSnapshot: null, + plannedDiff: null, + confirmationToken: null, + heartbeatAt: new Date().toISOString(), + leaseOwner: null, + leaseExpiresAt: null, + }); + await this.repository.appendSyncEvent(runId, "info", "succeeded", "Synchronization completed.", { ...applied }); + this.release(runId); + } catch (error) { + const current = await this.repository.getSyncRun(runId); + const cancelled = !timedOut && (error instanceof SyncCancelledError || controller.signal.aborted || current?.cancelRequested); + const failure = timedOut + ? { code: "schema_sync_timed_out", message: "Schema synchronization timed out." } + : safeFailure(error); + await this.repository.updateSyncRun(runId, { + state: cancelled ? "cancelled" : "failed", + phase: "completed", + errorCode: cancelled ? null : failure.code, + errorMessage: cancelled ? null : failure.message, + finishedAt: new Date().toISOString(), + observedSnapshot: null, + plannedDiff: null, + confirmationToken: null, + leaseOwner: null, + leaseExpiresAt: null, + }); + await this.repository.appendSyncEvent( + runId, + cancelled ? "warning" : "error", + cancelled ? "cancelled" : "failed", + cancelled ? "Synchronization cancelled." : failure.message, + ); + this.release(runId); + } finally { + clearTimeout(timeout); + clearInterval(heartbeat); + this.controllers.delete(runId); + } + } + + private assertReady(database: WorkspaceDatabase): void { + if (database.connectionStatus !== "reachable" || database.testedVersion !== database.version) { + throw new CatalogConflictError("Test the current database binding before synchronizing its schema"); + } + } + + private assertCapability(scope: CatalogSyncScope, snapshot: ObservedSchemaSnapshot): void { + const required = scope === "all" ? ["tables", "columns", "relationships"] as const : [scope] as const; + for (const name of required) { + if (snapshot.capabilities[name] !== "available") { + throw new CatalogSchemaCapabilityUnavailableError(name); + } + } + } + + private async checkCancelled(runId: string): Promise { + const run = await this.repository.getSyncRun(runId); + if (run?.cancelRequested) throw new SyncCancelledError("Synchronization cancelled"); + } + + private release(runId: string): void { + this.reservations.get(runId)?.(); + this.reservations.delete(runId); + } + + private phaseMessage(phase: Parameters[0]): string { + if (phase === "connecting") return "Connecting to the database."; + if (phase === "scanning_tables") return "Reading tables."; + if (phase === "scanning_columns") return "Reading columns and primary keys."; + return "Reading foreign-key relationships."; + } +} diff --git a/backend/src/catalog/table-introspector.ts b/backend/src/catalog/table-introspector.ts new file mode 100644 index 00000000..1761f6ab --- /dev/null +++ b/backend/src/catalog/table-introspector.ts @@ -0,0 +1,133 @@ +import { readFile } from "node:fs/promises"; +import type { WorkspaceSecretStore } from "../workspaces/secret-store.js"; +import type { CatalogPostgresAccess } from "./postgres-access.js"; +import { CATALOG_SECRET_IDS } from "./secrets.js"; +import { + CatalogConnectorError, + type ObservedCatalogTable, + type WorkspaceDatabase, +} from "./types.js"; + +export interface CatalogTableIntrospector { + scan(database: WorkspaceDatabase, signal: AbortSignal): Promise; +} + +function normalize(rows: readonly ObservedCatalogTable[]): ObservedCatalogTable[] { + const byName = new Map(); + for (const row of rows) { + if (typeof row.name !== "string" || row.name.length === 0 || row.name.length > 128) { + throw new CatalogConnectorError("Schema introspection response is invalid"); + } + if (row.sourceComment !== null && typeof row.sourceComment !== "string") { + throw new CatalogConnectorError("Schema introspection response is invalid"); + } + byName.set(row.name, row); + } + return [...byName.values()] + .sort((left, right) => left.name.localeCompare(right.name)); +} + +function requireText(value: string | undefined): string { + if (!value) throw new CatalogConnectorError("Database binding is incomplete"); + return value; +} + +export class ConcreteCatalogTableIntrospector implements CatalogTableIntrospector { + constructor( + private readonly postgres: CatalogPostgresAccess, + private readonly secretStore: WorkspaceSecretStore, + ) {} + + async scan(database: WorkspaceDatabase, signal: AbortSignal): Promise { + return database.binding.transport === "rest_api" + ? await this.scanRest(database, signal) + : await this.scanPostgres(database, signal); + } + + private async scanPostgres( + database: WorkspaceDatabase, + signal: AbortSignal, + ): Promise { + const client = await this.postgres.connect(database, signal); + try { + const schema = await client.query( + "SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_namespace WHERE nspname = $1) AS present", + [database.schema], + ); + if (schema.rows[0]?.present !== true) throw new CatalogConnectorError("Database schema is unavailable"); + const result = await client.query( + `SELECT c.relname AS name, d.description AS source_comment + FROM pg_catalog.pg_class c + JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace + LEFT JOIN pg_catalog.pg_description d ON d.objoid = c.oid AND d.objsubid = 0 + WHERE c.relkind IN ('r', 'p') AND n.nspname = $1 + ORDER BY c.relname`, + [database.schema], + ); + return normalize(result.rows.map((row) => ({ + name: String(row.name), + sourceComment: typeof row.source_comment === "string" && row.source_comment.length > 0 + ? row.source_comment + : null, + }))); + } finally { + await client.end(); + } + } + + private async scanRest( + database: WorkspaceDatabase, + signal: AbortSignal, + ): Promise { + const auth = database.binding.restAuth ?? "bearer"; + const required = auth === "none" ? [] : [CATALOG_SECRET_IDS.apiKey]; + const materialized = this.secretStore.materialize(database.workspaceId, required); + try { + const headers: Record = { "content-type": "application/json" }; + if (auth !== "none") { + const credentialFile = materialized.files.get(CATALOG_SECRET_IDS.apiKey); + if (!credentialFile) throw new CatalogConnectorError("REST API key is not configured"); + const credential = (await readFile(credentialFile, "utf8")).trim(); + if (auth === "bearer") headers.authorization = `Bearer ${credential}`; + else headers["x-api-key"] = credential; + } + const baseUrl = requireText(database.binding.baseUrl).replace(/\/+$/, ""); + const response = await fetch(`${baseUrl}/rpc/list_tables`, { + method: "POST", + headers, + body: JSON.stringify({ schema_name: database.schema }), + signal, + }); + if (!response.ok) throw new CatalogConnectorError("REST schema introspection failed"); + const body: unknown = await response.json(); + if (!Array.isArray(body)) throw new CatalogConnectorError("REST schema response is invalid"); + const rows: ObservedCatalogTable[] = []; + for (const item of body) { + if (!item || typeof item !== "object") { + throw new CatalogConnectorError("REST schema response is invalid"); + } + const row = item as Record; + if (typeof row.type !== "string") { + throw new CatalogConnectorError("REST schema response is invalid"); + } + if (row.type !== "TABLE") continue; + if (typeof row.table !== "string" || row.table.length === 0 || row.table.length > 128) { + throw new CatalogConnectorError("REST schema response is invalid"); + } + if (row.comment !== undefined && row.comment !== null && typeof row.comment !== "string") { + throw new CatalogConnectorError("REST schema response is invalid"); + } + rows.push({ + name: row.table, + sourceComment: typeof row.comment === "string" && row.comment.length > 0 ? row.comment : null, + }); + } + return normalize(rows); + } catch (error) { + if (error instanceof CatalogConnectorError) throw error; + throw new CatalogConnectorError("REST schema introspection failed"); + } finally { + materialized.release(); + } + } +} diff --git a/backend/src/catalog/table-service.ts b/backend/src/catalog/table-service.ts new file mode 100644 index 00000000..84cc3725 --- /dev/null +++ b/backend/src/catalog/table-service.ts @@ -0,0 +1,45 @@ +import type { + CatalogRepository, + CatalogTable, +} from "./types.js"; + +export class CatalogTableService { + constructor( + private readonly repository: CatalogRepository, + ) {} + + async list(databaseId: string): Promise { + return await this.repository.listTables(databaseId); + } + + async updateDescription( + databaseId: string, + tableId: string, + expectedVersion: number, + description: string | null, + ): Promise { + const normalized = description?.trim() || null; + return await this.repository.updateTableDescription( + databaseId, + tableId, + expectedVersion, + normalized, + ); + } + + async updateMetadata( + databaseId: string, + tableId: string, + expectedVersion: number, + description: string | null, + generatedDescription: string | null, + ): Promise { + return await this.repository.updateTableMetadata( + databaseId, + tableId, + expectedVersion, + description?.trim() || null, + generatedDescription?.trim() || null, + ); + } +} diff --git a/backend/src/catalog/types.ts b/backend/src/catalog/types.ts new file mode 100644 index 00000000..965dbbdf --- /dev/null +++ b/backend/src/catalog/types.ts @@ -0,0 +1,339 @@ +export const DATABASE_TRANSPORTS = ["postgres_direct", "rest_api", "ssh_tunnel"] as const; +export type DatabaseTransport = (typeof DATABASE_TRANSPORTS)[number]; + +export type ConnectionStatus = "untested" | "reachable" | "failed"; + +export interface DatabaseBinding { + transport: DatabaseTransport; + host?: string; + port?: number; + username?: string; + baseUrl?: string; + restPath?: string; + restAuth?: "none" | "bearer" | "x-api-key"; + tlsServername?: string; + sshHost?: string; + sshPort?: number; + sshUsername?: string; + sshTargetHost?: string; + sshTargetPort?: number; +} + +export interface WorkspaceDatabase { + id: string; + workspaceId: string; + engine: "postgres"; + databaseName: string; + schema: string; + version: number; + createdAt: string; + updatedAt: string; + binding: DatabaseBinding; + connectionStatus: ConnectionStatus; + testedVersion?: number; + lastTestedAt?: string; + lastErrorCode?: string; + lastErrorMessage?: string; + schemaSyncedVersion?: number; + schemaSyncedAt?: string; +} + +export interface DatabaseConfigurationInput { + workspaceId: string; + engine: "postgres"; + databaseName: string; + schema: string; + binding: DatabaseBinding; +} + +export interface DatabaseTestResult { + connectionStatus: Exclude; + testedVersion: number; + lastTestedAt: string; + errorCode?: string; + errorMessage?: string; +} + +export interface CatalogTable { + id: string; + databaseId: string; + name: string; + sourceComment: string | null; + description: string | null; + generatedDescription: string | null; + lastSyncedDatabaseVersion: number | null; + lastSyncedAt: string | null; + version: number; + createdAt: string; + updatedAt: string; +} + +export interface ObservedCatalogTable { + name: string; + sourceComment: string | null; +} + +export interface CatalogColumn { + id: string; + tableId: string; + name: string; + ordinalPosition: number; + dataType: string; + isNullable: boolean; + defaultExpression: string | null; + primaryKeyPosition: number | null; + isPrimaryKey: boolean; + isForeignKey: boolean; + foreignKeyCount: number; + sourceComment: string | null; + description: string | null; + generatedDescription: string | null; + lastSyncedDatabaseVersion: number | null; + lastSyncedAt: string | null; + version: number; + createdAt: string; + updatedAt: string; +} + +export interface ObservedCatalogColumn { + tableName: string; + name: string; + ordinalPosition: number; + dataType: string; + isNullable: boolean; + defaultExpression: string | null; + primaryKeyPosition: number | null; + sourceComment: string | null; +} + +export interface CatalogRelationshipColumn { + position: number; + sourceColumnId: string; + sourceColumnName: string; + targetColumnId: string; + targetColumnName: string; +} + +export interface CatalogRelationship { + id: string; + databaseId: string; + constraintName: string; + sourceTableId: string; + sourceTableName: string; + targetTableId: string; + targetTableName: string; + updateRule: string; + deleteRule: string; + deferrable: boolean; + initiallyDeferred: boolean; + columns: CatalogRelationshipColumn[]; + lastSyncedDatabaseVersion: number | null; + lastSyncedAt: string | null; + createdAt: string; + updatedAt: string; +} + +export interface ObservedRelationshipColumn { + position: number; + sourceColumnName: string; + targetColumnName: string; +} + +export interface ObservedCatalogRelationship { + constraintName: string; + sourceTableName: string; + targetTableName: string; + updateRule: string; + deleteRule: string; + deferrable: boolean; + initiallyDeferred: boolean; + columns: ObservedRelationshipColumn[]; +} + +export type IntrospectionCapabilityState = "available" | "unavailable"; +export interface ObservedSchemaSnapshot { + schemaVersion: 1; + capabilities: { + tables: IntrospectionCapabilityState; + columns: IntrospectionCapabilityState; + relationships: IntrospectionCapabilityState; + }; + tables: ObservedCatalogTable[]; + columns: ObservedCatalogColumn[]; + relationships: ObservedCatalogRelationship[]; +} + +export type CatalogSyncScope = "tables" | "columns" | "relationships" | "all"; +export type CatalogSyncState = + | "queued" | "running" | "awaiting_confirmation" | "applying" + | "succeeded" | "failed" | "cancelled" | "interrupted"; +export type CatalogSyncPhase = + | "queued" | "connecting" | "scanning_tables" | "scanning_columns" + | "scanning_relationships" | "planning" | "awaiting_confirmation" + | "applying" | "completed"; + +export interface CatalogSchemaDiff { + deletedTables: string[]; + deletedColumns: Array<{ tableName: string; columnName: string }>; + deletedRelationships: Array<{ sourceTableName: string; constraintName: string }>; +} + +export interface CatalogSyncCounts { + tables?: number; + columns?: number; + relationships?: number; + created?: number; + updated?: number; + deleted?: number; +} + +export interface CatalogSyncRun { + id: string; + databaseId: string; + scope: CatalogSyncScope; + tableIds: string[]; + state: CatalogSyncState; + phase: CatalogSyncPhase; + requestedDatabaseVersion: number; + observedSnapshot: ObservedSchemaSnapshot | null; + plannedDiff: CatalogSchemaDiff | null; + confirmationToken: string | null; + counts: CatalogSyncCounts; + errorCode: string | null; + errorMessage: string | null; + cancelRequested: boolean; + createdAt: string; + startedAt: string | null; + updatedAt: string; + finishedAt: string | null; + heartbeatAt: string | null; + leaseOwner: string | null; + leaseExpiresAt: string | null; +} + +export interface CatalogSyncEvent { + id: number; + runId: string; + sequence: number; + level: "info" | "warning" | "error"; + eventType: string; + message: string; + data: Record; + createdAt: string; +} + +export interface CatalogSyncRunUpdate { + state?: CatalogSyncState; + phase?: CatalogSyncPhase; + observedSnapshot?: ObservedSchemaSnapshot | null; + plannedDiff?: CatalogSchemaDiff | null; + confirmationToken?: string | null; + counts?: CatalogSyncCounts; + errorCode?: string | null; + errorMessage?: string | null; + cancelRequested?: boolean; + startedAt?: string | null; + finishedAt?: string | null; + heartbeatAt?: string | null; + leaseOwner?: string | null; + leaseExpiresAt?: string | null; +} + +export type TableSyncRepositoryResult = + | { kind: "confirmation_required"; deletedNames: string[] } + | { + kind: "applied"; + createdCount: number; + updatedCount: number; + deletedCount: number; + tables: CatalogTable[]; + }; + +export interface CatalogRepository { + list(): Promise; + get(id: string): Promise; + getByWorkspace(workspaceId: string): Promise; + create(input: DatabaseConfigurationInput): Promise; + update(id: string, expectedVersion: number, input: DatabaseConfigurationInput): Promise; + recordTest(id: string, expectedVersion: number, result: DatabaseTestResult): Promise; + touch(id: string, expectedVersion: number): Promise; + delete(id: string, expectedVersion: number): Promise; + listTables(databaseId: string): Promise; + getTable(databaseId: string, tableId: string): Promise; + updateTableDescription( + databaseId: string, + tableId: string, + expectedVersion: number, + description: string | null, + ): Promise; + updateTableMetadata( + databaseId: string, + tableId: string, + expectedVersion: number, + description: string | null, + generatedDescription: string | null, + ): Promise; + listColumns(databaseId: string, tableId: string): Promise; + getColumn(databaseId: string, tableId: string, columnId: string): Promise; + updateColumnMetadata( + databaseId: string, + tableId: string, + columnId: string, + expectedVersion: number, + description: string | null, + generatedDescription: string | null, + ): Promise; + listRelationships(databaseId: string): Promise; + planSchemaSync( + databaseId: string, + scope: CatalogSyncScope, + tableIds: readonly string[], + snapshot: ObservedSchemaSnapshot, + ): Promise; + applySchemaSync( + databaseId: string, + expectedDatabaseVersion: number, + scope: CatalogSyncScope, + tableIds: readonly string[], + snapshot: ObservedSchemaSnapshot, + ): Promise; + createSyncRun( + databaseId: string, + scope: CatalogSyncScope, + tableIds: readonly string[], + requestedDatabaseVersion: number, + ): Promise; + getSyncRun(runId: string): Promise; + claimSyncRun(runId: string, workerId: string, leaseExpiresAt: string): Promise; + listSyncRuns(databaseId: string, limit?: number): Promise; + updateSyncRun(runId: string, update: CatalogSyncRunUpdate): Promise; + requestSyncRunCancellation(runId: string): Promise; + appendSyncEvent( + runId: string, + level: CatalogSyncEvent["level"], + eventType: string, + message: string, + data?: Record, + ): Promise; + listSyncEvents(runId: string, afterSequence?: number): Promise; + pruneSyncEvents(before: string): Promise; + interruptActiveSyncRuns(): Promise; + reconcileTables( + databaseId: string, + expectedDatabaseVersion: number, + observed: readonly ObservedCatalogTable[], + confirmedDeletedNames: readonly string[], + ): Promise; + available(): Promise; + close?(): Promise; +} + +export class CatalogConflictError extends Error {} +export class CatalogUnavailableError extends Error {} +export class CatalogOperationInProgressError extends Error {} +export class CatalogConnectorError extends Error {} +export class CatalogSchemaCapabilityUnavailableError extends CatalogConnectorError { + constructor(readonly capability: "schema_snapshot" | "tables" | "columns" | "relationships") { + super(`Schema introspection capability '${capability}' is unavailable`); + } +} diff --git a/backend/src/config.ts b/backend/src/config.ts index 1b7e680a..1e6b9b06 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -7,6 +7,7 @@ import { } from "./auth/config.js"; import { createProjectedAuthenticationConfigProvider } from "./auth/runtime-projection.js"; import type { WorkspaceRegistryConfig } from "./workspaces/types.js"; +import type { CatalogConnectionConfig } from "./catalog/repository.js"; export interface AppConfig { host: string; port: number; harnessDir: string; thtBin: string; piBin: string; @@ -20,6 +21,8 @@ export interface AppConfig { host?: string; port?: number; database?: string; runtimeUser?: string; runtimePasswordFile?: string; sslmode?: "verify-ca" | "verify-full"; sslrootcert?: string; }; + /** Installation-local metadata catalog. Omitted installations expose an unavailable admin surface. */ + catalogDatabase?: CatalogConnectionConfig; defaults: { provider?: string; model?: string; thinking?: string }; maxPiProcesses: number; settingsFile: string; @@ -40,6 +43,7 @@ export interface AppConfig { /** Explicit compatibility mode for old loopback clients that send `workspace` in POST /sessions. */ legacyWorkspaceMode: boolean; workspaceDiagnosticTimeoutMs: number; + catalogSyncTimeoutMs: number; workspaceRegistry: WorkspaceRegistryConfig; workspaceSecretStoreRoot: string; workspaceSecretRuntimeRoot: string; @@ -127,6 +131,14 @@ function diagnosticTimeout(value: string | undefined): number { return timeout; } +function catalogSyncTimeout(value: string | undefined): number { + const timeout = Number(value ?? 600_000); + if (!Number.isSafeInteger(timeout) || timeout < 1_000 || timeout > 3_600_000) { + throw new Error("catalog synchronization timeout configuration is invalid"); + } + return timeout; +} + function piManagementTimeout(value: string | undefined): number { const timeout = Number(value ?? 8_000); if (!Number.isSafeInteger(timeout) || timeout < 1 || timeout > 30_000) { @@ -176,6 +188,33 @@ function positiveDimension(value: string | undefined, fallback: number): number return parsed; } +function catalogDatabase(env: Record): CatalogConnectionConfig | undefined { + const value = env.THT_CATALOG_DATABASE_URL; + if (value !== undefined) { + try { + const parsed = new URL(value); + if ((parsed.protocol !== "postgres:" && parsed.protocol !== "postgresql:") + || !parsed.hostname || !parsed.pathname.slice(1) || parsed.hash || parsed.search) throw new Error(); + return { connectionString: value }; + } catch { + throw new Error("catalog database configuration is invalid"); + } + } + const host = env.THT_CATALOG_DB_HOST; + if (host === undefined) return undefined; + const port = Number(env.THT_CATALOG_DB_PORT ?? 5432); + const database = env.THT_CATALOG_DB_NAME; + const user = env.THT_CATALOG_RUNTIME_USER; + const passwordFile = env.THT_CATALOG_RUNTIME_PASSWORD_FILE; + try { + if (!host.trim() || !database?.trim() || !user?.trim() || !passwordFile + || !path.isAbsolute(passwordFile) || !Number.isInteger(port) || port < 1 || port > 65_535) throw new Error(); + return { host, port, database, user, passwordFile }; + } catch { + throw new Error("catalog database configuration is invalid"); + } +} + export function loadConfig( env: Record, options: { surface?: "application" | "workspace-maintenance" } = {}, @@ -356,6 +395,7 @@ export function loadConfig( authentication, publicExposure, sessionStorage, + catalogDatabase: catalogDatabase(env), defaults: { provider: env.PI_PROVIDER, model: env.PI_MODEL, thinking: env.PI_THINKING }, maxPiProcesses: Number(env.MAX_PI_PROCESSES ?? 4), settingsFile, @@ -370,6 +410,7 @@ export function loadConfig( dwhPrecheck: env.THT_DWH_PRECHECK === "true" || env.THT_DWH_PRECHECK === "1", legacyWorkspaceMode: legacyWorkspaceMode === "local", workspaceDiagnosticTimeoutMs: diagnosticTimeout(env.THT_WORKSPACE_DIAGNOSTIC_TIMEOUT_MS), + catalogSyncTimeoutMs: catalogSyncTimeout(env.THT_CATALOG_SYNC_TIMEOUT_MS), workspaceRegistry, workspaceSecretStoreRoot, workspaceSecretRuntimeRoot, diff --git a/backend/src/routes/catalog-databases.ts b/backend/src/routes/catalog-databases.ts new file mode 100644 index 00000000..080a1008 --- /dev/null +++ b/backend/src/routes/catalog-databases.ts @@ -0,0 +1,218 @@ +import type { FastifyInstance, FastifyReply, FastifyRequest } from "fastify"; +import { z } from "zod"; +import { isPrincipalContext, requirePermission } from "../auth/authorization.js"; +import { CatalogService, type CatalogSecretName } from "../catalog/service.js"; +import { WorkspaceRegistryError } from "../workspaces/git-repository.js"; +import { + CatalogConflictError, + CatalogOperationInProgressError, + CatalogUnavailableError, + DATABASE_TRANSPORTS, + type CatalogRepository, + type DatabaseConfigurationInput, +} from "../catalog/types.js"; +import type { CatalogOperationCoordinator } from "../catalog/operation-coordinator.js"; + +const idSchema = z.uuid(); +const workspaceIdSchema = z.string().regex(/^[a-z][a-z0-9-]{2,62}$/); +const identifier = z.string().trim().min(1).max(128).regex(/^[A-Za-z_][A-Za-z0-9_$-]*$/); +const nonEmpty = z.string().trim().min(1).max(512); +const port = z.number().int().min(1).max(65_535); +const optionalText = nonEmpty.optional(); +const sshHost = z.string().trim().min(1).max(255).regex(/^[A-Za-z0-9_.:\[\]-]+$/).optional(); +const sshUsername = z.string().trim().min(1).max(128).regex(/^[A-Za-z0-9._-]+$/).optional(); +const bindingSchema = z.object({ + transport: z.enum(DATABASE_TRANSPORTS), + host: optionalText, + port: port.optional(), + username: optionalText, + baseUrl: z.url().max(2048).optional(), + restPath: z.string().regex(/^\/(?!\/)[^?#\\\u0000-\u001f]*$/).max(512).optional(), + restAuth: z.enum(["none", "bearer", "x-api-key"]).optional(), + tlsServername: optionalText, + sshHost, + sshPort: port.optional(), + sshUsername, + sshTargetHost: sshHost, + sshTargetPort: port.optional(), +}).strict().superRefine((binding, context) => { + const required = binding.transport === "postgres_direct" + ? ["host", "port", "username"] as const + : binding.transport === "rest_api" + ? ["baseUrl", "restPath", "restAuth"] as const + : ["username", "sshHost", "sshPort", "sshUsername", "sshTargetHost", "sshTargetPort"] as const; + for (const field of required) { + if (binding[field] === undefined) context.addIssue({ code: "custom", path: [field], message: "Required" }); + } +}); +const configSchema = z.object({ + workspaceId: workspaceIdSchema, + engine: z.literal("postgres"), + databaseName: identifier, + schema: identifier, + binding: bindingSchema, +}).strict(); +const updateSchema = configSchema.extend({ version: z.number().int().positive() }); +const secretNames = [ + "password", + "apiKey", + "sshPrivateKey", + "sshPrivateKeyPassphrase", + "sshKnownHosts", + "tlsCa", +] as const; +const secretsSchema = z.object({ + version: z.number().int().positive(), + values: z.partialRecord(z.enum(secretNames), z.string().min(1).max(65_536)).refine((values) => Object.keys(values).length > 0), +}).strict(); +const versionQuery = z.object({ version: z.coerce.number().int().positive() }); + +function safeError(reply: FastifyReply, error: unknown) { + if (error instanceof CatalogUnavailableError) { + return reply.code(503).send({ code: "catalog_unavailable", message: "Database catalog is unavailable." }); + } + if (error instanceof CatalogConflictError) { + return reply.code(409).send({ code: "database_conflict", message: "This workspace already has a database configuration." }); + } + if (error instanceof CatalogOperationInProgressError) { + return reply.code(409).send({ code: "database_operation_in_progress", message: "A database operation is already in progress." }); + } + if (error instanceof z.ZodError) { + return reply.code(400).send({ code: "database_invalid", message: "Database configuration is invalid." }); + } + if (error instanceof WorkspaceRegistryError) { + return reply.code(400).send({ code: "database_invalid", message: "Database configuration is invalid." }); + } + return reply.code(500).send({ code: "database_operation_failed", message: "Database operation failed." }); +} + +function manage(request: FastifyRequest, reply: FastifyReply) { + return isPrincipalContext(requirePermission(request, reply, "database.manage")); +} + +export function catalogDatabaseRoutes( + app: FastifyInstance, + deps: { repository: CatalogRepository; service: CatalogService; operations?: CatalogOperationCoordinator }, +): void { + const mutate = async (databaseId: string, operation: () => Promise): Promise => ( + deps.operations ? await deps.operations.run(databaseId, operation) : await operation() + ); + const activeSyncRun = async (databaseId: string) => { + const run = (await deps.repository.listSyncRuns(databaseId, 5)).find((candidate) => + ["queued", "running", "awaiting_confirmation", "applying"].includes(candidate.state)); + if (!run) return undefined; + const { + observedSnapshot: _snapshot, + leaseOwner: _leaseOwner, + leaseExpiresAt: _leaseExpiresAt, + ...summary + } = run; + return summary; + }; + app.get("/catalog/status", async (request, reply) => { + if (!manage(request, reply)) return reply; + return { available: await deps.repository.available() }; + }); + + app.get("/catalog/databases", async (request, reply) => { + if (!manage(request, reply)) return reply; + try { + const rows = await deps.service.list(); + return await Promise.all(rows.map(async (row) => ( + row.id ? { ...row, activeSyncRun: await activeSyncRun(row.id) } : row + ))); + } catch (error) { return safeError(reply, error); } + }); + + app.get("/catalog/databases/:id", async (request, reply) => { + if (!manage(request, reply)) return reply; + try { + const id = idSchema.parse((request.params as { id?: unknown }).id); + const database = await deps.repository.get(id); + if (!database) return reply.code(404).send({ code: "database_not_found", message: "Database configuration was not found." }); + return { + ...database, + configured: true, + secrets: deps.service.configuredSecrets(database.workspaceId), + activeSyncRun: await activeSyncRun(database.id), + }; + } catch (error) { return safeError(reply, error); } + }); + + app.post("/catalog/databases", async (request, reply) => { + if (!manage(request, reply)) return reply; + try { + const input = configSchema.parse(request.body) as DatabaseConfigurationInput; + const created = await deps.repository.create(await deps.service.normalizeInput(input)); + return reply.code(201).send({ ...created, configured: true, secrets: deps.service.configuredSecrets(created.workspaceId) }); + } catch (error) { return safeError(reply, error); } + }); + + app.patch("/catalog/databases/:id", async (request, reply) => { + if (!manage(request, reply)) return reply; + try { + const id = idSchema.parse((request.params as { id?: unknown }).id); + const { version, ...input } = updateSchema.parse(request.body); + const current = await deps.repository.get(id); + if (!current) return reply.code(404).send({ code: "database_not_found", message: "Database configuration was not found." }); + if (current.workspaceId !== input.workspaceId) { + return reply.code(400).send({ code: "database_invalid", message: "Database configuration is invalid." }); + } + const updated = await mutate(id, async () => await deps.repository.update( + id, version, await deps.service.normalizeInput(input as DatabaseConfigurationInput), + )); + if (!updated) return reply.code(409).send({ code: "database_stale", message: "Database configuration changed. Reload and try again." }); + return { ...updated, configured: true, secrets: deps.service.configuredSecrets(updated.workspaceId) }; + } catch (error) { return safeError(reply, error); } + }); + + app.put("/catalog/databases/:id/secrets", async (request, reply) => { + if (!manage(request, reply)) return reply; + if (!isPrincipalContext(requirePermission(request, reply, "workspace.secrets.manage"))) return reply; + try { + const id = idSchema.parse((request.params as { id?: unknown }).id); + const { version, values } = secretsSchema.parse(request.body); + const database = await deps.repository.get(id); + if (!database) return reply.code(404).send({ code: "database_not_found", message: "Database configuration was not found." }); + if (database.version !== version) return reply.code(409).send({ code: "database_stale", message: "Database configuration changed. Reload and try again." }); + const updated = await mutate(id, async () => { + const touched = await deps.repository.touch(id, version); + if (touched) deps.service.replaceSecrets(database.workspaceId, values as Partial>); + return touched; + }); + if (!updated) return reply.code(409).send({ code: "database_stale", message: "Database configuration changed. Reload and try again." }); + return { ...updated, configured: true, secrets: deps.service.configuredSecrets(updated.workspaceId) }; + } catch (error) { return safeError(reply, error); } + }); + + app.post("/catalog/databases/:id/test", async (request, reply) => { + if (!manage(request, reply)) return reply; + try { + const id = idSchema.parse((request.params as { id?: unknown }).id); + const { version } = z.object({ version: z.number().int().positive() }).strict().parse(request.body); + const database = await deps.repository.get(id); + if (!database) return reply.code(404).send({ code: "database_not_found", message: "Database configuration was not found." }); + if (database.version !== version) return reply.code(409).send({ code: "database_stale", message: "Database configuration changed. Reload and try again." }); + const tested = await deps.repository.recordTest(id, version, await deps.service.test(database)); + if (!tested) return reply.code(409).send({ code: "database_stale", message: "Database configuration changed. Reload and try again." }); + return { ...tested, configured: true, secrets: deps.service.configuredSecrets(tested.workspaceId) }; + } catch (error) { return safeError(reply, error); } + }); + + app.delete("/catalog/databases/:id", async (request, reply) => { + if (!manage(request, reply)) return reply; + try { + const id = idSchema.parse((request.params as { id?: unknown }).id); + const { version } = versionQuery.parse(request.query); + const database = await deps.repository.get(id); + if (!database) return reply.code(404).send({ code: "database_not_found", message: "Database configuration was not found." }); + const deleted = await mutate(id, async () => { + const removed = await deps.repository.delete(id, version); + if (removed) deps.service.forgetSecrets(database.workspaceId); + return removed; + }); + if (!deleted) return reply.code(409).send({ code: "database_stale", message: "Database configuration changed. Reload and try again." }); + return reply.code(204).send(); + } catch (error) { return safeError(reply, error); } + }); +} diff --git a/backend/src/routes/catalog-schema.ts b/backend/src/routes/catalog-schema.ts new file mode 100644 index 00000000..46948bad --- /dev/null +++ b/backend/src/routes/catalog-schema.ts @@ -0,0 +1,230 @@ +import type { FastifyInstance, FastifyReply, FastifyRequest } from "fastify"; +import { setTimeout as delay } from "node:timers/promises"; +import { z } from "zod"; +import { isPrincipalContext, requirePermission } from "../auth/authorization.js"; +import type { CatalogSyncWorker } from "../catalog/sync-worker.js"; +import { + CatalogConflictError, + CatalogConnectorError, + CatalogOperationInProgressError, + CatalogUnavailableError, + type CatalogRepository, + type CatalogSyncRun, +} from "../catalog/types.js"; + +const idSchema = z.uuid(); +const metadataSchema = z.object({ + version: z.number().int().positive(), + description: z.string().max(20_000).nullable(), + generatedDescription: z.string().max(20_000).nullable(), +}).strict(); +const createRunSchema = z.object({ + version: z.number().int().positive(), + scope: z.enum(["tables", "columns", "relationships", "all"]), + tableIds: z.array(idSchema).max(10_000).default([]), +}).strict(); +const confirmationSchema = z.object({ confirmationToken: z.string().uuid() }).strict(); +const eventQuerySchema = z.object({ after: z.coerce.number().int().nonnegative().default(0) }); + +function manage(request: FastifyRequest, reply: FastifyReply) { + return isPrincipalContext(requirePermission(request, reply, "database.manage")); +} + +function safeError(reply: FastifyReply, error: unknown) { + if (error instanceof CatalogUnavailableError) { + return reply.code(503).send({ code: "catalog_unavailable", message: "Database catalog is unavailable." }); + } + if (error instanceof CatalogConflictError || error instanceof CatalogOperationInProgressError) { + return reply.code(409).send({ code: "schema_sync_conflict", message: error.message }); + } + if (error instanceof CatalogConnectorError) { + return reply.code(502).send({ code: "schema_introspection_failed", message: "The database schema could not be read safely." }); + } + if (error instanceof z.ZodError) { + return reply.code(400).send({ code: "schema_request_invalid", message: "Schema request is invalid." }); + } + return reply.code(500).send({ code: "schema_operation_failed", message: "Schema operation failed." }); +} + +function normalized(value: string | null): string | null { + return value?.trim() || null; +} + +function publicRun(run: CatalogSyncRun) { + const { + observedSnapshot: _snapshot, + leaseOwner: _leaseOwner, + leaseExpiresAt: _leaseExpiresAt, + ...result + } = run; + return result; +} + +export function catalogSchemaRoutes( + app: FastifyInstance, + deps: { repository: CatalogRepository; worker: CatalogSyncWorker }, +): void { + app.get("/catalog/databases/:databaseId/tables/:tableId/columns", async (request, reply) => { + if (!manage(request, reply)) return reply; + try { + const params = request.params as { databaseId?: unknown; tableId?: unknown }; + const databaseId = idSchema.parse(params.databaseId); + const tableId = idSchema.parse(params.tableId); + if (!(await deps.repository.getTable(databaseId, tableId))) { + return reply.code(404).send({ code: "table_not_found", message: "Catalog table was not found." }); + } + return await deps.repository.listColumns(databaseId, tableId); + } catch (error) { return safeError(reply, error); } + }); + + app.patch("/catalog/databases/:databaseId/tables/:tableId/columns/:columnId", async (request, reply) => { + if (!manage(request, reply)) return reply; + try { + const params = request.params as { databaseId?: unknown; tableId?: unknown; columnId?: unknown }; + const databaseId = idSchema.parse(params.databaseId); + const tableId = idSchema.parse(params.tableId); + const columnId = idSchema.parse(params.columnId); + const input = metadataSchema.parse(request.body); + const current = await deps.repository.getColumn(databaseId, tableId, columnId); + if (!current) return reply.code(404).send({ code: "column_not_found", message: "Catalog column was not found." }); + if (current.version !== input.version) { + return reply.code(409).send({ code: "column_stale", message: "Column metadata changed. Reload and try again." }); + } + const updated = await deps.repository.updateColumnMetadata( + databaseId, + tableId, + columnId, + input.version, + normalized(input.description), + normalized(input.generatedDescription), + ); + if (!updated) return reply.code(409).send({ code: "column_stale", message: "Column metadata changed. Reload and try again." }); + return updated; + } catch (error) { return safeError(reply, error); } + }); + + app.get("/catalog/databases/:databaseId/relationships", async (request, reply) => { + if (!manage(request, reply)) return reply; + try { + const databaseId = idSchema.parse((request.params as { databaseId?: unknown }).databaseId); + if (!(await deps.repository.get(databaseId))) { + return reply.code(404).send({ code: "database_not_found", message: "Database configuration was not found." }); + } + return await deps.repository.listRelationships(databaseId); + } catch (error) { return safeError(reply, error); } + }); + + app.post("/catalog/databases/:databaseId/sync-runs", async (request, reply) => { + if (!manage(request, reply)) return reply; + try { + const databaseId = idSchema.parse((request.params as { databaseId?: unknown }).databaseId); + const input = createRunSchema.parse(request.body); + const database = await deps.repository.get(databaseId); + if (!database) return reply.code(404).send({ code: "database_not_found", message: "Database configuration was not found." }); + if (database.version !== input.version) { + return reply.code(409).send({ code: "database_stale", message: "Database configuration changed. Reload and try again." }); + } + return reply.code(202).send(publicRun(await deps.worker.start(database, input.scope, input.tableIds))); + } catch (error) { return safeError(reply, error); } + }); + + app.get("/catalog/databases/:databaseId/sync-runs", async (request, reply) => { + if (!manage(request, reply)) return reply; + try { + const databaseId = idSchema.parse((request.params as { databaseId?: unknown }).databaseId); + if (!(await deps.repository.get(databaseId))) { + return reply.code(404).send({ code: "database_not_found", message: "Database configuration was not found." }); + } + return (await deps.repository.listSyncRuns(databaseId)).map(publicRun); + } catch (error) { return safeError(reply, error); } + }); + + app.get("/catalog/sync-runs/:runId", async (request, reply) => { + if (!manage(request, reply)) return reply; + try { + const runId = idSchema.parse((request.params as { runId?: unknown }).runId); + const run = await deps.repository.getSyncRun(runId); + return run ? publicRun(run) : reply.code(404).send({ code: "sync_run_not_found", message: "Synchronization run was not found." }); + } catch (error) { return safeError(reply, error); } + }); + + app.post("/catalog/sync-runs/:runId/confirm", async (request, reply) => { + if (!manage(request, reply)) return reply; + try { + const runId = idSchema.parse((request.params as { runId?: unknown }).runId); + const { confirmationToken } = confirmationSchema.parse(request.body); + const run = await deps.worker.confirm(runId, confirmationToken); + return run ? publicRun(run) : reply.code(404).send({ code: "sync_run_not_found", message: "Synchronization run was not found." }); + } catch (error) { return safeError(reply, error); } + }); + + app.post("/catalog/sync-runs/:runId/cancel", async (request, reply) => { + if (!manage(request, reply)) return reply; + try { + const runId = idSchema.parse((request.params as { runId?: unknown }).runId); + const run = await deps.worker.cancel(runId); + return run ? publicRun(run) : reply.code(404).send({ code: "sync_run_not_found", message: "Synchronization run was not found." }); + } catch (error) { return safeError(reply, error); } + }); + + app.post("/catalog/sync-runs/:runId/retry", async (request, reply) => { + if (!manage(request, reply)) return reply; + try { + const runId = idSchema.parse((request.params as { runId?: unknown }).runId); + const run = await deps.worker.retry(runId); + return run ? reply.code(202).send(publicRun(run)) : reply.code(404).send({ code: "sync_run_not_found", message: "Synchronization run was not found." }); + } catch (error) { return safeError(reply, error); } + }); + + app.get("/catalog/sync-runs/:runId/events", async (request, reply) => { + if (!manage(request, reply)) return reply; + try { + const runId = idSchema.parse((request.params as { runId?: unknown }).runId); + let after = eventQuerySchema.parse(request.query).after; + const headerCursor = Number(request.headers["last-event-id"]); + if (Number.isInteger(headerCursor) && headerCursor >= 0) after = Math.max(after, headerCursor); + if (!(await deps.repository.getSyncRun(runId))) { + return reply.code(404).send({ code: "sync_run_not_found", message: "Synchronization run was not found." }); + } + reply.hijack(); + reply.raw.writeHead(200, { + "content-type": "text/event-stream; charset=utf-8", + "cache-control": "no-cache, no-transform", + connection: "keep-alive", + "x-accel-buffering": "no", + }); + const controller = new AbortController(); + request.raw.once("close", () => controller.abort()); + let lastRunUpdate = ""; + while (!controller.signal.aborted) { + const events = await deps.repository.listSyncEvents(runId, after); + for (const event of events) { + after = event.sequence; + reply.raw.write(`id: ${event.sequence}\nevent: log\ndata: ${JSON.stringify(event)}\n\n`); + } + const run = await deps.repository.getSyncRun(runId); + if (!run) break; + if (run.updatedAt !== lastRunUpdate) { + lastRunUpdate = run.updatedAt; + reply.raw.write(`event: run\ndata: ${JSON.stringify(publicRun(run))}\n\n`); + } + if (["succeeded", "failed", "cancelled", "interrupted"].includes(run.state)) break; + await delay(500, undefined, { signal: controller.signal }).catch(() => undefined); + } + reply.raw.end(); + return reply; + } catch (error) { return safeError(reply, error); } + }); + + app.get("/catalog/sync-runs/:runId/events-list", async (request, reply) => { + if (!manage(request, reply)) return reply; + try { + const runId = idSchema.parse((request.params as { runId?: unknown }).runId); + const after = eventQuerySchema.parse(request.query).after; + if (!(await deps.repository.getSyncRun(runId))) { + return reply.code(404).send({ code: "sync_run_not_found", message: "Synchronization run was not found." }); + } + return await deps.repository.listSyncEvents(runId, after); + } catch (error) { return safeError(reply, error); } + }); +} diff --git a/backend/src/routes/catalog-tables.ts b/backend/src/routes/catalog-tables.ts new file mode 100644 index 00000000..30baff3f --- /dev/null +++ b/backend/src/routes/catalog-tables.ts @@ -0,0 +1,79 @@ +import type { FastifyInstance, FastifyReply, FastifyRequest } from "fastify"; +import { z } from "zod"; +import { isPrincipalContext, requirePermission } from "../auth/authorization.js"; +import type { CatalogTableService } from "../catalog/table-service.js"; +import { + CatalogConnectorError, + CatalogOperationInProgressError, + CatalogUnavailableError, + type CatalogRepository, +} from "../catalog/types.js"; + +const idSchema = z.uuid(); +const updateSchema = z.object({ + version: z.number().int().positive(), + description: z.string().max(20_000).nullable(), + generatedDescription: z.string().max(20_000).nullable().optional(), +}).strict(); + +function manage(request: FastifyRequest, reply: FastifyReply) { + return isPrincipalContext(requirePermission(request, reply, "database.manage")); +} + +function safeError(reply: FastifyReply, error: unknown) { + if (error instanceof CatalogUnavailableError) { + return reply.code(503).send({ code: "catalog_unavailable", message: "Database catalog is unavailable." }); + } + if (error instanceof CatalogOperationInProgressError) { + return reply.code(409).send({ code: "database_operation_in_progress", message: "A database operation is already in progress." }); + } + if (error instanceof CatalogConnectorError) { + return reply.code(502).send({ code: "table_introspection_failed", message: "Database tables could not be read." }); + } + if (error instanceof z.ZodError) { + return reply.code(400).send({ code: "table_invalid", message: "Table request is invalid." }); + } + return reply.code(500).send({ code: "table_operation_failed", message: "Table operation failed." }); +} + +export function catalogTableRoutes( + app: FastifyInstance, + deps: { repository: CatalogRepository; service: CatalogTableService }, +): void { + app.get("/catalog/databases/:databaseId/tables", async (request, reply) => { + if (!manage(request, reply)) return reply; + try { + const databaseId = idSchema.parse((request.params as { databaseId?: unknown }).databaseId); + if (!(await deps.repository.get(databaseId))) { + return reply.code(404).send({ code: "database_not_found", message: "Database configuration was not found." }); + } + return await deps.service.list(databaseId); + } catch (error) { return safeError(reply, error); } + }); + + app.patch("/catalog/databases/:databaseId/tables/:tableId", async (request, reply) => { + if (!manage(request, reply)) return reply; + try { + const params = request.params as { databaseId?: unknown; tableId?: unknown }; + const databaseId = idSchema.parse(params.databaseId); + const tableId = idSchema.parse(params.tableId); + const input = updateSchema.parse(request.body); + const { version, description } = input; + const current = await deps.repository.getTable(databaseId, tableId); + if (!current) return reply.code(404).send({ code: "table_not_found", message: "Catalog table was not found." }); + if (current.version !== version) { + return reply.code(409).send({ code: "table_stale", message: "Table description changed. Reload and try again." }); + } + const updated = await deps.service.updateMetadata( + databaseId, + tableId, + version, + description, + input.generatedDescription === undefined ? current.generatedDescription : input.generatedDescription, + ); + if (!updated) return reply.code(409).send({ code: "table_stale", message: "Table description changed. Reload and try again." }); + return updated; + } catch (error) { return safeError(reply, error); } + }); + +} diff --git a/backend/test/auth-config.test.ts b/backend/test/auth-config.test.ts index 153755db..bb1f243b 100644 --- a/backend/test/auth-config.test.ts +++ b/backend/test/auth-config.test.ts @@ -187,6 +187,7 @@ test("roles collapse duplicates and admin contains all administrative permission "settings.manage", "workspace.manage", "workspace.secrets.manage", + "database.manage", "pi.manage", "auth.diagnostics.read", ]); diff --git a/backend/test/auth-request-snapshot.test.ts b/backend/test/auth-request-snapshot.test.ts index 548723a9..5a2064b2 100644 --- a/backend/test/auth-request-snapshot.test.ts +++ b/backend/test/auth-request-snapshot.test.ts @@ -114,7 +114,7 @@ test.each([ const created = await fixture.app.thothiiAuthSessionStore?.create({ principal: { issuer: "local", subject: user.id, displayName: user.username, roles: ["admin"], - permissions: ["session.use", "session.read_all", "session.manage_all", "settings.manage", "workspace.manage", "workspace.secrets.manage", "pi.manage", "auth.diagnostics.read"], + permissions: ["session.use", "session.read_all", "session.manage_all", "settings.manage", "workspace.manage", "workspace.secrets.manage", "database.manage", "pi.manage", "auth.diagnostics.read"], isAdmin: true, }, method: "local", diff --git a/backend/test/auth-routes-local.test.ts b/backend/test/auth-routes-local.test.ts index 8ff97319..27404c7b 100644 --- a/backend/test/auth-routes-local.test.ts +++ b/backend/test/auth-routes-local.test.ts @@ -130,7 +130,7 @@ test("local login sets a non-persistent opaque session cookie and exposes only a roles: ["admin"], permissions: [ "session.use", "session.read_all", "session.manage_all", "settings.manage", - "workspace.manage", "workspace.secrets.manage", "pi.manage", "auth.diagnostics.read", + "workspace.manage", "workspace.secrets.manage", "database.manage", "pi.manage", "auth.diagnostics.read", ], isAdmin: true, csrfToken: expect.stringMatching(/^[A-Za-z0-9_-]{43}$/), diff --git a/backend/test/auth.test.ts b/backend/test/auth.test.ts index 531d0243..c5a5379c 100644 --- a/backend/test/auth.test.ts +++ b/backend/test/auth.test.ts @@ -32,7 +32,7 @@ test("local mode resolves a stable local principal", async () => { roles: ["admin"], permissions: [ "session.use", "session.read_all", "session.manage_all", "settings.manage", - "workspace.manage", "workspace.secrets.manage", "pi.manage", "auth.diagnostics.read", + "workspace.manage", "workspace.secrets.manage", "database.manage", "pi.manage", "auth.diagnostics.read", ], isAdmin: true, }); @@ -74,7 +74,7 @@ test("upstream mode accepts only normalized proxy principal headers", async () = issuer: "portal", subject: "42", displayName: "Alice", roles: ["user", "admin"], permissions: [ "session.use", "session.read_all", "session.manage_all", "settings.manage", - "workspace.manage", "workspace.secrets.manage", "pi.manage", "auth.diagnostics.read", + "workspace.manage", "workspace.secrets.manage", "database.manage", "pi.manage", "auth.diagnostics.read", ], isAdmin: true, }); diff --git a/backend/test/authorization.test.ts b/backend/test/authorization.test.ts index cacfed39..f4471328 100644 --- a/backend/test/authorization.test.ts +++ b/backend/test/authorization.test.ts @@ -15,14 +15,14 @@ const admin: PrincipalContext = { issuer: "oidc", subject: "admin", roles: ["admin"], permissions: [ "session.use", "session.read_all", "session.manage_all", "settings.manage", - "workspace.manage", "workspace.secrets.manage", "pi.manage", "auth.diagnostics.read", + "workspace.manage", "workspace.secrets.manage", "database.manage", "pi.manage", "auth.diagnostics.read", ], isAdmin: true, }; const catalog: readonly Permission[] = [ "session.use", "session.read_all", "session.manage_all", "settings.manage", - "workspace.manage", "workspace.secrets.manage", "pi.manage", "auth.diagnostics.read", + "workspace.manage", "workspace.secrets.manage", "database.manage", "pi.manage", "auth.diagnostics.read", ]; test("permission matrix gives role-less identities no access, users session use, and admins every catalog permission", () => { diff --git a/backend/test/catalog-databases-routes.test.ts b/backend/test/catalog-databases-routes.test.ts new file mode 100644 index 00000000..f3d7002c --- /dev/null +++ b/backend/test/catalog-databases-routes.test.ts @@ -0,0 +1,122 @@ +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, expect, test, vi } from "vitest"; +import { buildApp } from "../src/app.js"; +import { loadConfig } from "../src/config.js"; +import { MemoryCatalogRepository } from "../src/catalog/memory-repository.js"; +import { WorkspaceSecretStore } from "../src/workspaces/secret-store.js"; +import type { WorkspaceRegistry, WorkspaceRevision } from "../src/workspaces/registry.js"; +import type { WorkspaceDescriptor } from "../src/workspaces/schema.js"; + +const roots: string[] = []; +afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); }); + +const workspace: WorkspaceDescriptor = { + workspace: { schema_version: 3, id: "psd-clinical", name: "Policlinico San Donato", language: "it" }, + dwh: { + engine: "postgres", database: "warehouse", schema: "datawarehouse", port: 5432, + supported_transports: ["postgres_direct", "rest_api"], + }, + semantic_index: { + vector_store: { engine: "qdrant", collection: "psd", dimensions: 1024, distance: "cosine" }, + embedding: { provider: "ollama_internal", model: "qwen3-embedding:0.6b", dimensions: 1024 }, + }, + llm_policy: { allowed: ["zai/glm-5.2"] }, + diagnostics: { dwh_rest: { method: "GET", path: "/health", auth: "bearer", response: { database: "database", schema: "schema" } } }, +}; +const revision: WorkspaceRevision = { id: "psd-clinical", commit: "a".repeat(40), blob: "b".repeat(40), snapshotPath: "/tmp/psd.yaml" }; + +function setup() { + const secretRoot = mkdtempSync(join(tmpdir(), "catalog-secret-")); + const runtimeRoot = mkdtempSync(join(tmpdir(), "catalog-secret-runtime-")); + roots.push(secretRoot, runtimeRoot); + const secretStore = new WorkspaceSecretStore({ root: secretRoot, runtimeRoot, installationId: "test" }); + const repository = new MemoryCatalogRepository(); + const registry = { + list: vi.fn(async () => [revision]), + listCatalog: vi.fn(async () => [{ id: "psd-clinical", name: "Policlinico San Donato", configurationState: "ready", revision }]), + read: vi.fn(async () => ({ workspace, revision })), + } as unknown as WorkspaceRegistry; + const app = buildApp(loadConfig({ THT_HARNESS_DIR: "/missing", NODE_ENV: "test" }), { + thtRunner: {} as never, + workspaceRegistry: registry, + workspaceSecretStore: secretStore, + catalogRepository: repository, + workspaceDiagnoser: vi.fn(), + }); + return { app, secretStore, repository }; +} + +const direct = { + workspaceId: "psd-clinical", + engine: "postgres", + databaseName: "warehouse", + schema: "datawarehouse", + binding: { transport: "postgres_direct", host: "db.internal", port: 5432, username: "reader" }, +}; + +test("lists every YAML workspace and creates its one database configuration", async () => { + const { app } = setup(); + const initial = await app.inject({ method: "GET", url: "/catalog/databases" }); + expect(initial.statusCode).toBe(200); + expect(initial.json()).toMatchObject([{ workspaceId: "psd-clinical", configured: false, databaseName: "warehouse" }]); + + const created = await app.inject({ method: "POST", url: "/catalog/databases", payload: direct }); + expect(created.statusCode).toBe(201); + expect(created.json()).toMatchObject({ configured: true, workspaceId: "psd-clinical", version: 1 }); + expect((await app.inject({ method: "POST", url: "/catalog/databases", payload: direct })).statusCode).toBe(409); + + const listed = await app.inject({ method: "GET", url: "/catalog/databases" }); + expect(listed.json()).toMatchObject([{ configured: true, binding: { transport: "postgres_direct", host: "db.internal" } }]); +}); + +test("lists orphaned records and takes the REST diagnostic path from workspace YAML", async () => { + const { app, repository } = setup(); + await repository.create({ + workspaceId: "removed-workspace", + engine: "postgres", + databaseName: "legacy", + schema: "public", + binding: { transport: "postgres_direct", host: "legacy.internal", port: 5432, username: "reader" }, + }); + const created = await app.inject({ + method: "POST", + url: "/catalog/databases", + payload: { + ...direct, + binding: { + transport: "rest_api", baseUrl: "https://psd.example/api", restPath: "/client-controlled", restAuth: "bearer", + }, + }, + }); + expect(created.statusCode).toBe(201); + expect(created.json()).toMatchObject({ binding: { restPath: "/health" } }); + + const rows = (await app.inject({ method: "GET", url: "/catalog/databases" })).json(); + expect(rows).toEqual(expect.arrayContaining([ + expect.objectContaining({ workspaceId: "removed-workspace", configured: true, workspaceAvailable: false }), + expect.objectContaining({ workspaceId: "psd-clinical", configured: true, workspaceAvailable: true }), + ])); +}); + +test("uses optimistic versions, keeps secrets write-only, and hard-deletes only local configuration", async () => { + const { app, secretStore } = setup(); + const created = (await app.inject({ method: "POST", url: "/catalog/databases", payload: direct })).json(); + const stale = await app.inject({ method: "PATCH", url: `/catalog/databases/${created.id}`, payload: { ...direct, version: 99 } }); + expect(stale.statusCode).toBe(409); + + const secret = await app.inject({ + method: "PUT", url: `/catalog/databases/${created.id}/secrets`, + payload: { version: 1, values: { password: "do-not-return-this" } }, + }); + expect(secret.statusCode).toBe(200); + expect(secret.body).not.toContain("do-not-return-this"); + expect(secret.json()).toMatchObject({ version: 2, secrets: { password: true } }); + expect(secretStore.has("psd-clinical", "catalog.dwh.password")).toBe(true); + + const removed = await app.inject({ method: "DELETE", url: `/catalog/databases/${created.id}?version=2` }); + expect(removed.statusCode).toBe(204); + expect(secretStore.has("psd-clinical", "catalog.dwh.password")).toBe(false); + expect((await app.inject({ method: "GET", url: "/catalog/databases" })).json()).toMatchObject([{ configured: false }]); +}); diff --git a/backend/test/catalog-postgres-access.test.ts b/backend/test/catalog-postgres-access.test.ts new file mode 100644 index 00000000..dfe9adf5 --- /dev/null +++ b/backend/test/catalog-postgres-access.test.ts @@ -0,0 +1,171 @@ +import { EventEmitter } from "node:events"; +import { existsSync, mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { PassThrough } from "node:stream"; +import type { ChildProcessWithoutNullStreams } from "node:child_process"; +import type { Client, ClientConfig } from "pg"; +import { afterEach, expect, test, vi } from "vitest"; +import { + buildSshArguments, + ConcreteCatalogPostgresAccess, +} from "../src/catalog/postgres-access.js"; +import { CATALOG_SECRET_IDS } from "../src/catalog/secrets.js"; +import type { WorkspaceDatabase } from "../src/catalog/types.js"; +import { WorkspaceSecretStore } from "../src/workspaces/secret-store.js"; + +const roots: string[] = []; + +afterEach(() => { + for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); +}); + +function secretStore() { + const root = mkdtempSync(join(tmpdir(), "catalog-ssh-secrets-")); + const runtimeRoot = mkdtempSync(join(tmpdir(), "catalog-ssh-runtime-")); + roots.push(root, runtimeRoot); + return new WorkspaceSecretStore({ root, runtimeRoot, installationId: "test" }); +} + +function sshDatabase(): WorkspaceDatabase { + return { + id: "11111111-1111-4111-8111-111111111111", + workspaceId: "psd-clinical", + engine: "postgres", + databaseName: "warehouse", + schema: "datawarehouse", + version: 4, + createdAt: "2026-08-27T08:00:00Z", + updatedAt: "2026-08-27T09:00:00Z", + connectionStatus: "reachable", + binding: { + transport: "ssh_tunnel", + username: "warehouse_reader", + sshHost: "bastion.internal", + sshPort: 2222, + sshUsername: "tunnel_user", + sshTargetHost: "postgres.internal", + sshTargetPort: 5432, + tlsServername: "postgres.internal", + }, + }; +} + +function fakeChild(): ChildProcessWithoutNullStreams { + const child = new EventEmitter() as EventEmitter & { + stdin: PassThrough; + stdout: PassThrough; + stderr: PassThrough; + exitCode: number | null; + signalCode: NodeJS.Signals | null; + kill: (signal?: NodeJS.Signals | number) => boolean; + }; + child.stdin = new PassThrough(); + child.stdout = new PassThrough(); + child.stderr = new PassThrough(); + child.exitCode = null; + child.signalCode = null; + child.kill = vi.fn((signal: NodeJS.Signals | number = "SIGTERM") => { + child.signalCode = typeof signal === "string" ? signal : "SIGTERM"; + child.emit("exit", null, child.signalCode); + return true; + }); + return child as unknown as ChildProcessWithoutNullStreams; +} + +test("builds a strict host-verified OpenSSH stdio tunnel", () => { + const args = buildSshArguments({ + sshHost: "bastion.internal", + sshPort: 2222, + sshUsername: "tunnel_user", + targetHost: "postgres.internal", + targetPort: 5432, + privateKeyFile: "/runtime/id", + knownHostsFile: "/runtime/known_hosts", + passphraseFile: "/runtime/passphrase", + connectTimeoutMs: 5_001, + }); + + expect(args).toEqual(expect.arrayContaining([ + "-F", "/dev/null", + "-o", "BatchMode=no", + "-o", "StrictHostKeyChecking=yes", + "-o", "UserKnownHostsFile=/runtime/known_hosts", + "-o", "GlobalKnownHostsFile=/dev/null", + "-o", "IdentitiesOnly=yes", + "-o", "IdentityAgent=none", + "-o", "PasswordAuthentication=no", + "-o", "KbdInteractiveAuthentication=no", + "-o", "ConnectTimeout=6", + "-W", "postgres.internal:5432", + "--", "tunnel_user@bastion.internal", + ])); +}); + +test("connects pg through OpenSSH, supplies askpass, and releases all secret leases", async () => { + const store = secretStore(); + store.putMany("psd-clinical", { + [CATALOG_SECRET_IDS.password]: "db-password ", + [CATALOG_SECRET_IDS.sshPrivateKey]: "PRIVATE KEY\n", + [CATALOG_SECRET_IDS.sshPrivateKeyPassphrase]: "key-passphrase", + [CATALOG_SECRET_IDS.sshKnownHosts]: "bastion.internal ssh-ed25519 AAAATEST\n", + [CATALOG_SECRET_IDS.tlsCa]: "CA CERTIFICATE\n", + }); + const child = fakeChild(); + let clientConfig: ClientConfig | undefined; + let spawnCall: { command: string; args: readonly string[]; env: NodeJS.ProcessEnv } | undefined; + const end = vi.fn(async () => undefined); + const query = vi.fn(async () => ({ rows: [{ ok: true }] })); + const connect = vi.fn(async () => undefined); + + const access = new ConcreteCatalogPostgresAccess(store, { + sshBinary: "/usr/bin/ssh", + askpassPath: "/app/ssh-askpass.mjs", + connectTimeoutMs: 5_000, + spawnSsh: (command, args, options) => { + spawnCall = { command, args, env: options.env }; + return child; + }, + createClient: (config) => { + clientConfig = config; + return { connect, query, end } as unknown as Client; + }, + }); + + const client = await access.connect(sshDatabase(), new AbortController().signal); + expect(connect).toHaveBeenCalledOnce(); + expect(clientConfig).toMatchObject({ + host: "postgres.internal", + port: 5432, + database: "warehouse", + user: "warehouse_reader", + password: "db-password ", + connectionTimeoutMillis: 5_000, + ssl: { + ca: "CA CERTIFICATE\n", + servername: "postgres.internal", + rejectUnauthorized: true, + }, + }); + expect(clientConfig?.stream).toBeTypeOf("function"); + expect(spawnCall?.command).toBe("/usr/bin/ssh"); + expect(spawnCall?.args.some((argument) => argument.startsWith("IdentityFile="))).toBe(true); + expect(spawnCall?.args.some((argument) => argument.startsWith("UserKnownHostsFile="))).toBe(true); + expect(spawnCall?.env).toMatchObject({ + DISPLAY: "thothii", + SSH_ASKPASS: "/app/ssh-askpass.mjs", + SSH_ASKPASS_REQUIRE: "force", + }); + const leasedPaths = spawnCall!.args + .filter((argument) => argument.startsWith("IdentityFile=") || argument.startsWith("UserKnownHostsFile=")) + .map((argument) => argument.slice(argument.indexOf("=") + 1)); + leasedPaths.push(spawnCall!.env.THT_SSH_PASSPHRASE_FILE!); + expect(leasedPaths.every(existsSync)).toBe(true); + + await expect(client.query("SELECT 1", [])).resolves.toEqual({ rows: [{ ok: true }] }); + await client.end(); + + expect(end).toHaveBeenCalledOnce(); + expect(child.kill).toHaveBeenCalledWith("SIGTERM"); + expect(leasedPaths.some(existsSync)).toBe(false); +}); diff --git a/backend/test/catalog-repository.integration.test.ts b/backend/test/catalog-repository.integration.test.ts new file mode 100644 index 00000000..a76644bf --- /dev/null +++ b/backend/test/catalog-repository.integration.test.ts @@ -0,0 +1,125 @@ +import { spawnSync } from "node:child_process"; +import { PostgreSqlContainer } from "@testcontainers/postgresql"; +import { CamelCasePlugin, Kysely, PostgresDialect, sql } from "kysely"; +import { Pool } from "pg"; +import { expect, test } from "vitest"; +import { KyselyCatalogRepository, type CatalogDatabase } from "../src/catalog/repository.js"; +import type { ObservedSchemaSnapshot } from "../src/catalog/types.js"; +import { up as upDatabases } from "../src/catalog/migrations/001_workspace_databases.js"; +import { up as upTables } from "../src/catalog/migrations/002_catalog_tables.js"; +import { up as upSchemaSync } from "../src/catalog/migrations/003_catalog_schema_sync.js"; +import { up as upRuntimeSequencePrivileges } from "../src/catalog/migrations/004_catalog_runtime_sequence_privileges.js"; + +const dockerAvailable = spawnSync("docker", ["info"], { stdio: "ignore" }).status === 0; + +test.skipIf(!dockerAvailable)("PostgreSQL migration enforces one database per workspace and optimistic updates", async () => { + const container = await new PostgreSqlContainer("postgres:17.6-bookworm").start(); + const db = new Kysely({ + dialect: new PostgresDialect({ pool: new Pool({ connectionString: container.getConnectionUri() }) }), + plugins: [new CamelCasePlugin()], + }); + try { + await upDatabases(db); + await upTables(db); + await upSchemaSync(db); + await sql`CREATE ROLE thothii_catalog_runtime`.execute(db); + await upRuntimeSequencePrivileges(db); + const sequencePrivilege = await sql<{ allowed: boolean }>` + SELECT has_sequence_privilege( + 'thothii_catalog_runtime', + 'catalog_sync_events_id_seq', + 'USAGE' + ) AS allowed + `.execute(db); + expect(sequencePrivilege.rows[0]?.allowed).toBe(true); + const repository = new KyselyCatalogRepository(db); + const input = { + workspaceId: "psd-clinical", + engine: "postgres" as const, + databaseName: "warehouse", + schema: "datawarehouse", + binding: { transport: "rest_api" as const, baseUrl: "https://psd.example/api", restPath: "/health", restAuth: "bearer" as const }, + }; + const created = await repository.create(input); + expect(created).toMatchObject({ version: 1, connectionStatus: "untested", binding: { transport: "rest_api" } }); + await expect(repository.create(input)).rejects.toThrow("Workspace database already exists"); + expect(await repository.update(created.id, 99, input)).toBeUndefined(); + const synchronized = await repository.reconcileTables(created.id, 1, [ + { name: "patients", sourceComment: "Clinical patients" }, + { name: "visits", sourceComment: null }, + ], []); + expect(synchronized).toMatchObject({ kind: "applied", createdCount: 2, deletedCount: 0 }); + const patients = (await repository.listTables(created.id))[0]; + expect(await repository.updateTableDescription( + created.id, + patients.id, + patients.version, + "Curated patients", + )).toMatchObject({ description: "Curated patients", sourceComment: "Clinical patients", version: 2 }); + const visits = (await repository.listTables(created.id)).find((table) => table.name === "visits")!; + const fullColumnsSnapshot: ObservedSchemaSnapshot = { + schemaVersion: 1, + capabilities: { tables: "available", columns: "available", relationships: "available" }, + tables: [ + { name: "patients", sourceComment: "Clinical patients" }, + { name: "visits", sourceComment: null }, + ], + columns: [ + { tableName: "patients", name: "id", ordinalPosition: 1, dataType: "bigint", isNullable: false, defaultExpression: null, primaryKeyPosition: 1, sourceComment: null }, + { tableName: "patients", name: "name", ordinalPosition: 2, dataType: "text", isNullable: true, defaultExpression: null, primaryKeyPosition: null, sourceComment: null }, + { tableName: "visits", name: "id", ordinalPosition: 1, dataType: "bigint", isNullable: false, defaultExpression: null, primaryKeyPosition: 1, sourceComment: null }, + { tableName: "visits", name: "patient_id", ordinalPosition: 2, dataType: "bigint", isNullable: false, defaultExpression: null, primaryKeyPosition: null, sourceComment: null }, + ], + relationships: [], + }; + expect(await repository.applySchemaSync(created.id, 1, "columns", [], fullColumnsSnapshot)) + .toMatchObject({ created: 4, deleted: 0 }); + expect((await repository.listColumns(created.id, patients.id)).map((column) => column.name)) + .toEqual(["id", "name"]); + expect((await repository.listColumns(created.id, visits.id)).map((column) => column.name)) + .toEqual(["id", "patient_id"]); + + const reducedColumnsSnapshot: ObservedSchemaSnapshot = { + ...fullColumnsSnapshot, + columns: fullColumnsSnapshot.columns.filter((column) => column.name === "id"), + }; + expect(await repository.planSchemaSync(created.id, "columns", [], reducedColumnsSnapshot)).toMatchObject({ + deletedColumns: [ + { tableName: "patients", columnName: "name" }, + { tableName: "visits", columnName: "patient_id" }, + ], + }); + expect(await repository.planSchemaSync(created.id, "columns", [patients.id], reducedColumnsSnapshot)).toMatchObject({ + deletedColumns: [{ tableName: "patients", columnName: "name" }], + }); + expect(await repository.applySchemaSync(created.id, 1, "columns", [patients.id], reducedColumnsSnapshot)) + .toMatchObject({ deleted: 1 }); + expect((await repository.listColumns(created.id, patients.id)).map((column) => column.name)) + .toEqual(["id"]); + expect((await repository.listColumns(created.id, visits.id)).map((column) => column.name)) + .toEqual(["id", "patient_id"]); + expect(await repository.reconcileTables(created.id, 1, [ + { name: "patients", sourceComment: "Updated physical comment" }, + ], [])).toEqual({ kind: "confirmation_required", deletedNames: ["visits"] }); + expect(await repository.reconcileTables(created.id, 1, [ + { name: "patients", sourceComment: "Updated physical comment" }, + ], ["visits"])).toMatchObject({ kind: "applied", updatedCount: 1, deletedCount: 1 }); + const syncRun = await repository.createSyncRun(created.id, "columns", [patients.id], 1); + expect(syncRun).toMatchObject({ + databaseId: created.id, + scope: "columns", + tableIds: [patients.id], + state: "queued", + }); + expect(await repository.listSyncRuns(created.id)).toEqual([ + expect.objectContaining({ id: syncRun.id, tableIds: [patients.id] }), + ]); + expect(await repository.update(created.id, 1, { ...input, schema: "public" })).toMatchObject({ version: 2, schema: "public" }); + expect(await repository.delete(created.id, 2)).toBe(true); + expect(await repository.list()).toEqual([]); + expect(await repository.listTables(created.id)).toEqual([]); + } finally { + await db.destroy(); + await container.stop(); + } +}, 60_000); diff --git a/backend/test/catalog-schema-introspector.test.ts b/backend/test/catalog-schema-introspector.test.ts new file mode 100644 index 00000000..dab74d02 --- /dev/null +++ b/backend/test/catalog-schema-introspector.test.ts @@ -0,0 +1,194 @@ +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, expect, test, vi } from "vitest"; +import type { CatalogDatabaseClient, CatalogPostgresAccess } from "../src/catalog/postgres-access.js"; +import { ConcreteCatalogSchemaIntrospector } from "../src/catalog/schema-introspector.js"; +import { + CatalogSchemaCapabilityUnavailableError, + type WorkspaceDatabase, +} from "../src/catalog/types.js"; +import { WorkspaceSecretStore } from "../src/workspaces/secret-store.js"; + +const roots: string[] = []; +afterEach(() => { + vi.unstubAllGlobals(); + for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); +}); + +function store() { + const root = mkdtempSync(join(tmpdir(), "catalog-schema-introspection-secrets-")); + const runtimeRoot = mkdtempSync(join(tmpdir(), "catalog-schema-introspection-runtime-")); + roots.push(root, runtimeRoot); + return new WorkspaceSecretStore({ root, runtimeRoot, installationId: "test" }); +} + +function database(binding: WorkspaceDatabase["binding"]): WorkspaceDatabase { + return { + id: "11111111-1111-4111-8111-111111111111", + workspaceId: "psd-clinical", + engine: "postgres", + databaseName: "warehouse", + schema: "datawarehouse", + binding, + version: 4, + connectionStatus: "reachable", + testedVersion: 4, + createdAt: "2026-08-27T08:00:00Z", + updatedAt: "2026-08-27T09:00:00Z", + }; +} + +test("reads columns, ordered composite keys, and physical relationships from one PostgreSQL connection", async () => { + const query = vi.fn() + .mockResolvedValueOnce({ rows: [{ present: true }] }) + .mockResolvedValueOnce({ rows: [{ name: "visits", source_comment: "Visits" }] }) + .mockResolvedValueOnce({ rows: [ + { table_name: "visits", name: "tenant_id", ordinal_position: 1, data_type: "uuid", is_nullable: false, default_expression: null, primary_key_position: 1, source_comment: null }, + { table_name: "visits", name: "patient_id", ordinal_position: 2, data_type: "bigint", is_nullable: false, default_expression: null, primary_key_position: 2, source_comment: "Patient" }, + ] }) + .mockResolvedValueOnce({ rows: [ + { constraint_name: "visits_patient_fkey", source_table_name: "visits", target_table_name: "patients", update_action: "a", delete_action: "c", deferrable: true, initially_deferred: false, position: 1, source_column_name: "tenant_id", target_column_name: "tenant_id" }, + { constraint_name: "visits_patient_fkey", source_table_name: "visits", target_table_name: "patients", update_action: "a", delete_action: "c", deferrable: true, initially_deferred: false, position: 2, source_column_name: "patient_id", target_column_name: "id" }, + ] }); + const end = vi.fn(async () => undefined); + const client: CatalogDatabaseClient = { query, end }; + const postgres: CatalogPostgresAccess = { connect: vi.fn(async () => client) }; + const introspector = new ConcreteCatalogSchemaIntrospector(postgres, store()); + + const result = await introspector.scan(database({ + transport: "ssh_tunnel", + username: "reader", + sshHost: "bastion.internal", + sshPort: 22, + sshUsername: "tunnel", + sshTargetHost: "db.internal", + sshTargetPort: 5432, + }), new AbortController().signal); + + expect(result.capabilities).toEqual({ tables: "available", columns: "available", relationships: "available" }); + expect(result.columns).toMatchObject([ + { name: "tenant_id", primaryKeyPosition: 1, isNullable: false }, + { name: "patient_id", primaryKeyPosition: 2, sourceComment: "Patient" }, + ]); + expect(result.relationships).toEqual([expect.objectContaining({ + constraintName: "visits_patient_fkey", + updateRule: "NO ACTION", + deleteRule: "CASCADE", + deferrable: true, + columns: [ + { position: 1, sourceColumnName: "tenant_id", targetColumnName: "tenant_id" }, + { position: 2, sourceColumnName: "patient_id", targetColumnName: "id" }, + ], + })]); + expect(query.mock.calls[2][0]).toContain("format_type"); + expect(query.mock.calls[3][0]).toContain("WITH ORDINALITY"); + expect(query.mock.calls.slice(1).every((call) => call[1][0] === "datawarehouse")).toBe(true); + expect(end).toHaveBeenCalledOnce(); +}); + +test("uses the typed full REST snapshot RPC and preserves explicit capability unavailability", async () => { + const response = { + schemaVersion: 1, + capabilities: { tables: "available", columns: "unavailable", relationships: "unavailable" }, + tables: [{ name: "patients", sourceComment: null }], + columns: [], + relationships: [], + }; + const fetchMock = vi.fn(async () => new Response(JSON.stringify(response), { status: 200, headers: { "content-type": "application/json" } })); + vi.stubGlobal("fetch", fetchMock); + const postgres: CatalogPostgresAccess = { connect: vi.fn(async () => { throw new Error("wire access must not be used"); }) }; + const introspector = new ConcreteCatalogSchemaIntrospector(postgres, store()); + + const result = await introspector.scan(database({ + transport: "rest_api", baseUrl: "https://connector.internal/api/", restPath: "/health", restAuth: "none", + }), new AbortController().signal); + + expect(result).toEqual(response); + expect(fetchMock).toHaveBeenCalledWith( + "https://connector.internal/api/rpc/schema_snapshot", + expect.objectContaining({ method: "POST", body: JSON.stringify({ schema_name: "datawarehouse" }) }), + ); +}); + +test("falls back to one read-only REST query when the snapshot RPC is absent", async () => { + const response = { + schemaVersion: 1 as const, + capabilities: { tables: "available" as const, columns: "available" as const, relationships: "available" as const }, + tables: [ + { name: "patients", sourceComment: "Clinical patients" }, + { name: "visits", sourceComment: null }, + ], + columns: [ + { tableName: "patients", name: "tenant_id", ordinalPosition: 1, dataType: "uuid", isNullable: false, defaultExpression: null, primaryKeyPosition: 1, sourceComment: "Tenant" }, + { tableName: "patients", name: "id", ordinalPosition: 2, dataType: "bigint", isNullable: false, defaultExpression: "nextval('patients_id_seq'::regclass)", primaryKeyPosition: 2, sourceComment: null }, + { tableName: "visits", name: "tenant_id", ordinalPosition: 1, dataType: "uuid", isNullable: false, defaultExpression: null, primaryKeyPosition: null, sourceComment: null }, + { tableName: "visits", name: "patient_id", ordinalPosition: 2, dataType: "bigint", isNullable: true, defaultExpression: null, primaryKeyPosition: null, sourceComment: "Owning patient" }, + ], + relationships: [{ + constraintName: "visits_patient_fkey", + sourceTableName: "visits", + targetTableName: "patients", + updateRule: "CASCADE", + deleteRule: "RESTRICT", + deferrable: true, + initiallyDeferred: false, + columns: [ + { position: 1, sourceColumnName: "tenant_id", targetColumnName: "tenant_id" }, + { position: 2, sourceColumnName: "patient_id", targetColumnName: "id" }, + ], + }], + }; + const fetchMock = vi.fn() + .mockResolvedValueOnce(new Response(null, { status: 404 })) + .mockResolvedValueOnce(new Response(JSON.stringify([response]), { + status: 200, + headers: { "content-type": "application/json" }, + })); + vi.stubGlobal("fetch", fetchMock); + const postgres: CatalogPostgresAccess = { + connect: vi.fn(async () => { throw new Error("wire access must not be used"); }), + }; + const introspector = new ConcreteCatalogSchemaIntrospector(postgres, store()); + + const result = await introspector.scan(database({ + transport: "rest_api", + baseUrl: "https://connector.internal/api/", + restPath: "/health", + restAuth: "none", + }), new AbortController().signal); + + expect(result).toEqual(response); + expect(fetchMock).toHaveBeenCalledTimes(2); + expect(fetchMock.mock.calls[0]).toEqual([ + "https://connector.internal/api/rpc/schema_snapshot", + expect.objectContaining({ method: "POST", body: JSON.stringify({ schema_name: "datawarehouse" }) }), + ]); + expect(fetchMock.mock.calls[1][0]).toBe("https://connector.internal/api/rpc/run_query"); + const fallbackRequest = fetchMock.mock.calls[1][1] as RequestInit; + expect(fallbackRequest).toMatchObject({ method: "POST" }); + const fallbackBody = JSON.parse(String(fallbackRequest.body)) as { query_text: string }; + expect(Object.keys(fallbackBody)).toEqual(["query_text"]); + expect(fallbackBody.query_text).toMatch(/^\s*WITH\b/); + expect(fallbackBody.query_text).toContain("pg_catalog.pg_constraint"); + expect(fallbackBody.query_text).not.toMatch(/\b(INSERT|UPDATE|DROP|ALTER|CREATE|TRUNCATE)\b/i); +}); + +test("classifies a missing REST snapshot RPC as an explicit binding capability", async () => { + vi.stubGlobal("fetch", vi.fn(async () => new Response(null, { status: 404 }))); + const postgres: CatalogPostgresAccess = { + connect: vi.fn(async () => { throw new Error("wire access must not be used"); }), + }; + const introspector = new ConcreteCatalogSchemaIntrospector(postgres, store()); + + const scan = introspector.scan(database({ + transport: "rest_api", + baseUrl: "https://connector.internal/api/", + restPath: "/health", + restAuth: "none", + }), new AbortController().signal); + + await expect(scan).rejects.toMatchObject({ + capability: "schema_snapshot", + }); +}); diff --git a/backend/test/catalog-schema-routes.test.ts b/backend/test/catalog-schema-routes.test.ts new file mode 100644 index 00000000..d112be0e --- /dev/null +++ b/backend/test/catalog-schema-routes.test.ts @@ -0,0 +1,224 @@ +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, expect, test, vi } from "vitest"; +import { buildApp } from "../src/app.js"; +import { loadConfig } from "../src/config.js"; +import { MemoryCatalogRepository } from "../src/catalog/memory-repository.js"; +import type { CatalogSchemaIntrospector } from "../src/catalog/schema-introspector.js"; +import type { CatalogSyncRun, ObservedSchemaSnapshot } from "../src/catalog/types.js"; +import { WorkspaceSecretStore } from "../src/workspaces/secret-store.js"; +import type { WorkspaceRegistry, WorkspaceRevision } from "../src/workspaces/registry.js"; +import type { WorkspaceDescriptor } from "../src/workspaces/schema.js"; + +const roots: string[] = []; +afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); }); + +const workspace: WorkspaceDescriptor = { + workspace: { schema_version: 3, id: "psd-clinical", name: "Policlinico San Donato", language: "it" }, + dwh: { engine: "postgres", database: "warehouse", schema: "datawarehouse", port: 5432, supported_transports: ["postgres_direct"] }, + semantic_index: { + vector_store: { engine: "qdrant", collection: "psd", dimensions: 1024, distance: "cosine" }, + embedding: { provider: "ollama_internal", model: "qwen3-embedding:0.6b", dimensions: 1024 }, + }, + llm_policy: { allowed: ["zai/glm-5.2"] }, +}; +const revision: WorkspaceRevision = { id: "psd-clinical", commit: "a".repeat(40), blob: "b".repeat(40), snapshotPath: "/tmp/psd.yaml" }; + +function snapshot(): ObservedSchemaSnapshot { + return { + schemaVersion: 1, + capabilities: { tables: "available", columns: "available", relationships: "available" }, + tables: [ + { name: "patients", sourceComment: "Clinical patients" }, + { name: "visits", sourceComment: "Patient visits" }, + ], + columns: [ + { tableName: "patients", name: "id", ordinalPosition: 1, dataType: "bigint", isNullable: false, defaultExpression: null, primaryKeyPosition: 1, sourceComment: "Patient key" }, + { tableName: "visits", name: "id", ordinalPosition: 1, dataType: "bigint", isNullable: false, defaultExpression: null, primaryKeyPosition: 1, sourceComment: null }, + { tableName: "visits", name: "patient_id", ordinalPosition: 2, dataType: "bigint", isNullable: false, defaultExpression: null, primaryKeyPosition: null, sourceComment: "Owning patient" }, + ], + relationships: [{ + constraintName: "visits_patient_id_fkey", + sourceTableName: "visits", + targetTableName: "patients", + updateRule: "NO ACTION", + deleteRule: "CASCADE", + deferrable: false, + initiallyDeferred: false, + columns: [{ position: 1, sourceColumnName: "patient_id", targetColumnName: "id" }], + }], + }; +} + +async function waitFor(repository: MemoryCatalogRepository, runId: string, state: CatalogSyncRun["state"]): Promise { + for (let attempt = 0; attempt < 100; attempt += 1) { + const run = await repository.getSyncRun(runId); + if (run?.state === state) return run; + await new Promise((resolve) => setTimeout(resolve, 5)); + } + throw new Error(`Run ${runId} did not reach ${state}`); +} + +async function setup() { + const secretRoot = mkdtempSync(join(tmpdir(), "catalog-schema-secret-")); + const runtimeRoot = mkdtempSync(join(tmpdir(), "catalog-schema-runtime-")); + roots.push(secretRoot, runtimeRoot); + const repository = new MemoryCatalogRepository(); + const created = await repository.create({ + workspaceId: "psd-clinical", engine: "postgres", databaseName: "warehouse", schema: "datawarehouse", + binding: { transport: "postgres_direct", host: "db.internal", port: 5432, username: "reader" }, + }); + await repository.recordTest(created.id, created.version, { + connectionStatus: "reachable", testedVersion: created.version, lastTestedAt: new Date().toISOString(), + }); + let observed = snapshot(); + const scan = vi.fn(async (_database, _signal, progress) => { + await progress?.("connecting"); + await progress?.("scanning_tables", { tables: observed.tables.length }); + await progress?.("scanning_columns", { tables: observed.tables.length, columns: observed.columns.length }); + await progress?.("scanning_relationships", { relationships: observed.relationships.length }); + return structuredClone(observed); + }); + const introspector: CatalogSchemaIntrospector = { scan }; + const registry = { + list: vi.fn(async () => [revision]), + listCatalog: vi.fn(async () => [{ id: "psd-clinical", name: "Policlinico San Donato", configurationState: "ready", revision }]), + read: vi.fn(async () => ({ workspace, revision })), + } as unknown as WorkspaceRegistry; + const app = buildApp(loadConfig({ THT_HARNESS_DIR: "/missing", NODE_ENV: "test" }), { + thtRunner: {} as never, + workspaceRegistry: registry, + workspaceSecretStore: new WorkspaceSecretStore({ root: secretRoot, runtimeRoot, installationId: "test" }), + catalogRepository: repository, + catalogSchemaIntrospector: introspector, + workspaceDiagnoser: vi.fn(), + }); + return { + app, repository, database: (await repository.get(created.id))!, scan, + setObserved(next: ObservedSchemaSnapshot) { observed = next; }, + }; +} + +test("synchronizes a full physical schema and derives primary and foreign key flags", async () => { + const { app, repository, database } = await setup(); + const started = await app.inject({ + method: "POST", url: `/catalog/databases/${database.id}/sync-runs`, + payload: { version: database.version, scope: "all", tableIds: [] }, + }); + expect(started.statusCode).toBe(202); + const completed = await waitFor(repository, started.json().id, "succeeded"); + expect(completed.counts).toMatchObject({ tables: 2, columns: 3, relationships: 1 }); + + const tables = await repository.listTables(database.id); + const visits = tables.find((table) => table.name === "visits")!; + const columns = (await app.inject({ method: "GET", url: `/catalog/databases/${database.id}/tables/${visits.id}/columns` })).json(); + expect(columns).toMatchObject([ + { name: "id", isPrimaryKey: true, primaryKeyPosition: 1, isForeignKey: false }, + { name: "patient_id", isPrimaryKey: false, isForeignKey: true, foreignKeyCount: 1 }, + ]); + const relationships = (await app.inject({ method: "GET", url: `/catalog/databases/${database.id}/relationships` })).json(); + expect(relationships).toMatchObject([{ constraintName: "visits_patient_id_fkey", columns: [{ sourceColumnName: "patient_id", targetColumnName: "id" }] }]); + expect((await repository.get(database.id))?.schemaSyncedVersion).toBe(database.version); +}); + +test("synchronizes columns for every catalog table when no table selection is supplied", async () => { + const { app, repository, database, setObserved } = await setup(); + const tablesRun = await app.inject({ + method: "POST", + url: `/catalog/databases/${database.id}/sync-runs`, + payload: { version: database.version, scope: "tables", tableIds: [] }, + }); + expect(tablesRun.statusCode).toBe(202); + await waitFor(repository, tablesRun.json().id, "succeeded"); + const tables = await repository.listTables(database.id); + expect(tables.map((table) => table.name)).toEqual(["patients", "visits"]); + + const columnsRun = await app.inject({ + method: "POST", + url: `/catalog/databases/${database.id}/sync-runs`, + payload: { version: database.version, scope: "columns", tableIds: [] }, + }); + expect(columnsRun.statusCode).toBe(202); + await waitFor(repository, columnsRun.json().id, "succeeded"); + const patients = tables.find((table) => table.name === "patients")!; + const visits = tables.find((table) => table.name === "visits")!; + expect((await repository.listColumns(database.id, patients.id)).map((column) => column.name)).toEqual(["id"]); + expect((await repository.listColumns(database.id, visits.id)).map((column) => column.name)).toEqual(["id", "patient_id"]); + + const next = snapshot(); + next.columns = next.columns.filter((column) => column.name !== "id"); + setObserved(next); + const selectedDestructiveRun = await app.inject({ + method: "POST", + url: `/catalog/databases/${database.id}/sync-runs`, + payload: { version: database.version, scope: "columns", tableIds: [patients.id] }, + }); + expect(selectedDestructiveRun.statusCode).toBe(202); + const selectedWaiting = await waitFor(repository, selectedDestructiveRun.json().id, "awaiting_confirmation"); + expect(selectedWaiting.plannedDiff?.deletedColumns).toEqual([ + { tableName: "patients", columnName: "id" }, + ]); + expect((await app.inject({ + method: "POST", + url: `/catalog/sync-runs/${selectedWaiting.id}/cancel`, + })).statusCode).toBe(200); + + const destructiveRun = await app.inject({ + method: "POST", + url: `/catalog/databases/${database.id}/sync-runs`, + payload: { version: database.version, scope: "columns", tableIds: [] }, + }); + expect(destructiveRun.statusCode).toBe(202); + const waiting = await waitFor(repository, destructiveRun.json().id, "awaiting_confirmation"); + expect(waiting.plannedDiff?.deletedColumns).toEqual([ + { tableName: "patients", columnName: "id" }, + { tableName: "visits", columnName: "id" }, + ]); +}); + +test("keeps generated descriptions editable and preserves them across synchronization", async () => { + const { app, repository, database } = await setup(); + const first = await app.inject({ method: "POST", url: `/catalog/databases/${database.id}/sync-runs`, payload: { version: database.version, scope: "all", tableIds: [] } }); + await waitFor(repository, first.json().id, "succeeded"); + const patients = (await repository.listTables(database.id)).find((table) => table.name === "patients")!; + const editedTable = await app.inject({ + method: "PATCH", url: `/catalog/databases/${database.id}/tables/${patients.id}`, + payload: { version: patients.version, description: null, generatedDescription: "Generated table draft" }, + }); + expect(editedTable.json()).toMatchObject({ description: null, generatedDescription: "Generated table draft" }); + const idColumn = (await repository.listColumns(database.id, patients.id))[0]; + const editedColumn = await app.inject({ + method: "PATCH", url: `/catalog/databases/${database.id}/tables/${patients.id}/columns/${idColumn.id}`, + payload: { version: idColumn.version, description: "Reviewed key", generatedDescription: "Generated key draft" }, + }); + expect(editedColumn.json()).toMatchObject({ description: "Reviewed key", generatedDescription: "Generated key draft" }); + + const second = await app.inject({ method: "POST", url: `/catalog/databases/${database.id}/sync-runs`, payload: { version: database.version, scope: "all", tableIds: [] } }); + await waitFor(repository, second.json().id, "succeeded"); + expect(await repository.getTable(database.id, patients.id)).toMatchObject({ generatedDescription: "Generated table draft" }); + expect(await repository.getColumn(database.id, patients.id, idColumn.id)).toMatchObject({ description: "Reviewed key", generatedDescription: "Generated key draft" }); +}); + +test("waits for confirmation and rescans before applying destructive changes", async () => { + const { app, repository, database, scan, setObserved } = await setup(); + const first = await app.inject({ method: "POST", url: `/catalog/databases/${database.id}/sync-runs`, payload: { version: database.version, scope: "all", tableIds: [] } }); + await waitFor(repository, first.json().id, "succeeded"); + const next = snapshot(); + next.tables = next.tables.filter((table) => table.name !== "visits"); + next.columns = next.columns.filter((column) => column.tableName !== "visits"); + next.relationships = []; + setObserved(next); + + const destructive = await app.inject({ method: "POST", url: `/catalog/databases/${database.id}/sync-runs`, payload: { version: database.version, scope: "all", tableIds: [] } }); + const waiting = await waitFor(repository, destructive.json().id, "awaiting_confirmation"); + expect(waiting.plannedDiff).toMatchObject({ deletedTables: ["visits"] }); + expect(await repository.listTables(database.id)).toHaveLength(2); + const confirmed = await app.inject({ + method: "POST", url: `/catalog/sync-runs/${waiting.id}/confirm`, payload: { confirmationToken: waiting.confirmationToken }, + }); + expect(confirmed.statusCode).toBe(200); + await waitFor(repository, waiting.id, "succeeded"); + expect((await repository.listTables(database.id)).map((table) => table.name)).toEqual(["patients"]); + expect(scan).toHaveBeenCalledTimes(3); +}); diff --git a/backend/test/catalog-table-introspector.test.ts b/backend/test/catalog-table-introspector.test.ts new file mode 100644 index 00000000..20042948 --- /dev/null +++ b/backend/test/catalog-table-introspector.test.ts @@ -0,0 +1,124 @@ +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, expect, test, vi } from "vitest"; +import type { + CatalogDatabaseClient, + CatalogPostgresAccess, +} from "../src/catalog/postgres-access.js"; +import { CATALOG_SECRET_IDS } from "../src/catalog/secrets.js"; +import { ConcreteCatalogTableIntrospector } from "../src/catalog/table-introspector.js"; +import type { WorkspaceDatabase } from "../src/catalog/types.js"; +import { WorkspaceSecretStore } from "../src/workspaces/secret-store.js"; + +const roots: string[] = []; + +afterEach(() => { + vi.unstubAllGlobals(); + for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); +}); + +function secretStore() { + const root = mkdtempSync(join(tmpdir(), "catalog-table-introspection-secrets-")); + const runtimeRoot = mkdtempSync(join(tmpdir(), "catalog-table-introspection-runtime-")); + roots.push(root, runtimeRoot); + return new WorkspaceSecretStore({ root, runtimeRoot, installationId: "test" }); +} + +function database(binding: WorkspaceDatabase["binding"]): WorkspaceDatabase { + return { + id: "11111111-1111-4111-8111-111111111111", + workspaceId: "psd-clinical", + engine: "postgres", + databaseName: "warehouse", + schema: "datawarehouse", + version: 4, + createdAt: "2026-08-27T08:00:00Z", + updatedAt: "2026-08-27T09:00:00Z", + connectionStatus: "reachable", + binding, + }; +} + +test("reads only ordinary and partitioned PostgreSQL tables from the configured schema", async () => { + const query = vi.fn() + .mockResolvedValueOnce({ rows: [{ present: true }] }) + .mockResolvedValueOnce({ rows: [ + { name: "visits", source_comment: null }, + { name: "patients", source_comment: "Clinical patients" }, + ] }); + const end = vi.fn(async () => undefined); + const client: CatalogDatabaseClient = { query, end }; + const postgres: CatalogPostgresAccess = { connect: vi.fn(async () => client) }; + const introspector = new ConcreteCatalogTableIntrospector(postgres, secretStore()); + + const tables = await introspector.scan(database({ + transport: "postgres_direct", + host: "db.internal", + port: 5432, + username: "reader", + }), new AbortController().signal); + + expect(tables).toEqual([ + { name: "patients", sourceComment: "Clinical patients" }, + { name: "visits", sourceComment: null }, + ]); + expect(query.mock.calls[1][0]).toContain("c.relkind IN ('r', 'p')"); + expect(query.mock.calls[1][0]).not.toContain("'v'"); + expect(query.mock.calls[1][1]).toEqual(["datawarehouse"]); + expect(end).toHaveBeenCalledOnce(); +}); + +test("uses the typed REST table RPC and ignores non-table objects", async () => { + const store = secretStore(); + store.put("psd-clinical", CATALOG_SECRET_IDS.apiKey, "rest-secret"); + const fetchMock = vi.fn(async () => new Response(JSON.stringify([ + { type: "VIEW", table: "patient_view", comment: "Not a table" }, + { type: "TABLE", table: "visits", comment: null }, + { type: "TABLE", table: "patients", comment: "Clinical patients" }, + ]), { status: 200, headers: { "content-type": "application/json" } })); + vi.stubGlobal("fetch", fetchMock); + const postgres: CatalogPostgresAccess = { + connect: vi.fn(async () => { throw new Error("PostgreSQL wire access must not be used"); }), + }; + const introspector = new ConcreteCatalogTableIntrospector(postgres, store); + + const tables = await introspector.scan(database({ + transport: "rest_api", + baseUrl: "https://connector.internal/api/", + restPath: "/health", + restAuth: "x-api-key", + }), new AbortController().signal); + + expect(tables).toEqual([ + { name: "patients", sourceComment: "Clinical patients" }, + { name: "visits", sourceComment: null }, + ]); + expect(fetchMock).toHaveBeenCalledWith( + "https://connector.internal/api/rpc/list_tables", + expect.objectContaining({ + method: "POST", + headers: expect.objectContaining({ "x-api-key": "rest-secret" }), + body: JSON.stringify({ schema_name: "datawarehouse" }), + }), + ); +}); + +test("fails closed when a REST table row violates the typed contract", async () => { + const store = secretStore(); + const fetchMock = vi.fn(async () => new Response(JSON.stringify([ + { type: "TABLE", table_name: "patients", comment: "Wrong field name" }, + ]), { status: 200, headers: { "content-type": "application/json" } })); + vi.stubGlobal("fetch", fetchMock); + const postgres: CatalogPostgresAccess = { + connect: vi.fn(async () => { throw new Error("PostgreSQL wire access must not be used"); }), + }; + const introspector = new ConcreteCatalogTableIntrospector(postgres, store); + + await expect(introspector.scan(database({ + transport: "rest_api", + baseUrl: "https://connector.internal/api", + restPath: "/health", + restAuth: "none", + }), new AbortController().signal)).rejects.toThrow("REST schema response is invalid"); +}); diff --git a/backend/test/catalog-tables-routes.test.ts b/backend/test/catalog-tables-routes.test.ts new file mode 100644 index 00000000..f787f6d1 --- /dev/null +++ b/backend/test/catalog-tables-routes.test.ts @@ -0,0 +1,126 @@ +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, expect, test, vi } from "vitest"; +import { buildApp } from "../src/app.js"; +import { loadConfig } from "../src/config.js"; +import { MemoryCatalogRepository } from "../src/catalog/memory-repository.js"; +import { WorkspaceSecretStore } from "../src/workspaces/secret-store.js"; +import type { WorkspaceRegistry, WorkspaceRevision } from "../src/workspaces/registry.js"; +import type { WorkspaceDescriptor } from "../src/workspaces/schema.js"; + +const roots: string[] = []; +afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); }); + +const workspace: WorkspaceDescriptor = { + workspace: { schema_version: 3, id: "psd-clinical", name: "Policlinico San Donato", language: "it" }, + dwh: { + engine: "postgres", database: "warehouse", schema: "datawarehouse", port: 5432, + supported_transports: ["postgres_direct", "rest_api"], + }, + semantic_index: { + vector_store: { engine: "qdrant", collection: "psd", dimensions: 1024, distance: "cosine" }, + embedding: { provider: "ollama_internal", model: "qwen3-embedding:0.6b", dimensions: 1024 }, + }, + llm_policy: { allowed: ["zai/glm-5.2"] }, + diagnostics: { dwh_rest: { method: "GET", path: "/health", auth: "bearer", response: { database: "database", schema: "schema" } } }, +}; +const revision: WorkspaceRevision = { + id: "psd-clinical", commit: "a".repeat(40), blob: "b".repeat(40), snapshotPath: "/tmp/psd.yaml", +}; + +async function setup() { + const secretRoot = mkdtempSync(join(tmpdir(), "catalog-table-secret-")); + const runtimeRoot = mkdtempSync(join(tmpdir(), "catalog-table-runtime-")); + roots.push(secretRoot, runtimeRoot); + const repository = new MemoryCatalogRepository(); + const database = await repository.create({ + workspaceId: "psd-clinical", + engine: "postgres", + databaseName: "warehouse", + schema: "datawarehouse", + binding: { transport: "postgres_direct", host: "db.internal", port: 5432, username: "reader" }, + }); + await repository.recordTest(database.id, database.version, { + connectionStatus: "reachable", + testedVersion: database.version, + lastTestedAt: new Date().toISOString(), + }); + const scan = vi.fn(async () => [ + { name: "patients", sourceComment: "Clinical patients" }, + { name: "visits", sourceComment: null }, + ]); + const registry = { + list: vi.fn(async () => [revision]), + listCatalog: vi.fn(async () => [{ id: "psd-clinical", name: "Policlinico San Donato", configurationState: "ready", revision }]), + read: vi.fn(async () => ({ workspace, revision })), + } as unknown as WorkspaceRegistry; + const secretStore = new WorkspaceSecretStore({ root: secretRoot, runtimeRoot, installationId: "test" }); + const app = buildApp(loadConfig({ THT_HARNESS_DIR: "/missing", NODE_ENV: "test" }), { + thtRunner: {} as never, + workspaceRegistry: registry, + workspaceSecretStore: secretStore, + catalogRepository: repository, + workspaceDiagnoser: vi.fn(), + }); + return { app, repository, database: (await repository.get(database.id))!, scan }; +} + +test("lists physical tables and updates only review metadata", async () => { + const { app, repository, database, scan } = await setup(); + const synchronized = await repository.reconcileTables(database.id, database.version, await scan(), []); + expect(synchronized).toMatchObject({ kind: "applied", createdCount: 2, deletedCount: 0 }); + expect(scan).toHaveBeenCalledOnce(); + + const tables = (await app.inject({ + method: "GET", url: `/catalog/databases/${database.id}/tables`, + })).json(); + expect(tables.map((table: { name: string }) => table.name)).toEqual(["patients", "visits"]); + const patients = tables[0]; + const edited = await app.inject({ + method: "PATCH", + url: `/catalog/databases/${database.id}/tables/${patients.id}`, + payload: { version: patients.version, description: "Curated patient registry" }, + }); + expect(edited.statusCode).toBe(200); + expect(edited.json()).toMatchObject({ + name: "patients", + sourceComment: "Clinical patients", + description: "Curated patient registry", + version: 2, + }); +}); + +test("requires an exact deletion confirmation before applying the atomic diff", async () => { + const { app, repository, database, scan } = await setup(); + await repository.reconcileTables(database.id, database.version, await scan(), []); + scan.mockResolvedValue([{ name: "patients", sourceComment: "Clinical patients" }]); + + const preview = await repository.reconcileTables(database.id, database.version, await scan(), []); + expect(preview).toEqual({ kind: "confirmation_required", deletedNames: ["visits"] }); + expect((await app.inject({ method: "GET", url: `/catalog/databases/${database.id}/tables` })).json()).toHaveLength(2); + + const applied = await repository.reconcileTables(database.id, database.version, await scan(), ["visits"]); + expect(applied).toMatchObject({ kind: "applied", deletedCount: 1 }); + expect((await app.inject({ method: "GET", url: `/catalog/databases/${database.id}/tables` })).json()).toMatchObject([ + { name: "patients" }, + ]); +}); + +test("refuses synchronization until the current binding has passed its connection test", async () => { + const { app, repository, database } = await setup(); + await repository.update(database.id, database.version, { + workspaceId: database.workspaceId, + engine: database.engine, + databaseName: database.databaseName, + schema: database.schema, + binding: database.binding, + }); + const response = await app.inject({ + method: "POST", + url: `/catalog/databases/${database.id}/sync-runs`, + payload: { version: database.version + 1, scope: "tables", tableIds: [] }, + }); + expect(response.statusCode).toBe(409); + expect(response.json()).toMatchObject({ code: "schema_sync_conflict" }); +}); diff --git a/backend/test/config.test.ts b/backend/test/config.test.ts index 17d8ab6f..aa5ca8e4 100644 --- a/backend/test/config.test.ts +++ b/backend/test/config.test.ts @@ -289,3 +289,22 @@ test("loadConfig accepts only an absolute generic model key file", () => { expect(() => loadConfig({ THT_MODEL_API_KEY_FILE: " /run/secrets/key" })) .toThrow(/model credential configuration is invalid/); }); + +test("loadConfig accepts a file-backed catalog role and rejects partial catalog configuration", () => { + expect(loadConfig({ + THT_CATALOG_DB_HOST: "catalog-db", + THT_CATALOG_DB_NAME: "thothii_catalog", + THT_CATALOG_RUNTIME_USER: "thothii_catalog_runtime", + THT_CATALOG_RUNTIME_PASSWORD_FILE: "/run/secrets/catalog_runtime_password", + }).catalogDatabase).toEqual({ + host: "catalog-db", + port: 5432, + database: "thothii_catalog", + user: "thothii_catalog_runtime", + passwordFile: "/run/secrets/catalog_runtime_password", + }); + expect(() => loadConfig({ THT_CATALOG_DB_HOST: "catalog-db" })) + .toThrow(/catalog database configuration is invalid/); + expect(() => loadConfig({ THT_CATALOG_DATABASE_URL: "https://catalog.invalid/db" })) + .toThrow(/catalog database configuration is invalid/); +}); diff --git a/compose.yaml b/compose.yaml index f395f575..642c9fc6 100644 --- a/compose.yaml +++ b/compose.yaml @@ -25,6 +25,11 @@ services: THT_PI_AUTH_FILE: /home/thoth/.pi/agent/auth.json THT_AUTH_CONFIG_FILE: /run/thothii-auth/auth.yaml THT_AUTH_STATE_ROOT: /data/auth + THT_CATALOG_DB_HOST: catalog-db + THT_CATALOG_DB_PORT: "5432" + THT_CATALOG_DB_NAME: thothii_catalog + THT_CATALOG_RUNTIME_USER: thothii_catalog_runtime + THT_CATALOG_RUNTIME_PASSWORD_FILE: /run/secrets/catalog_runtime_password THT_DB_NAME: ${THT_DB_NAME:-} THT_DWH_REST_URL: ${THT_DWH_REST_URL:-} THT_LLM_URL: ${THT_LLM_URL:-} @@ -47,6 +52,7 @@ services: secrets: - source: thothii_secrets target: thothii.secrets + - catalog_runtime_password healthcheck: test: ["CMD", "curl", "-fsS", "http://127.0.0.1:8787/health"] interval: 15s @@ -54,6 +60,8 @@ services: retries: 5 start_period: 30s depends_on: + catalog-db: + condition: service_healthy qdrant: condition: service_healthy embedding-model-init: @@ -61,6 +69,51 @@ services: networks: - thothii + catalog-db: + image: postgres:17.6-bookworm@sha256:f3bd19c606e442c3d7bdfa8002e03fe260a1023351e0ea4598032022b68dd6e3 + environment: + POSTGRES_DB: thothii_catalog + POSTGRES_USER: thothii_catalog_migrate + POSTGRES_PASSWORD_FILE: /run/secrets/catalog_migrator_password + volumes: + - catalog-data:/var/lib/postgresql/data + - ./docker/catalog-db-init.sql:/docker-entrypoint-initdb.d/010-runtime-role.sql:ro + secrets: + - catalog_runtime_password + - catalog_migrator_password + healthcheck: + test: + - CMD-SHELL + - >- + pg_isready -U thothii_catalog_migrate -d thothii_catalog + && test "$(psql -U thothii_catalog_migrate -d thothii_catalog -Atqc + "select count(*) from pg_catalog.pg_roles where rolname = 'thothii_catalog_runtime'")" = "1" + interval: 5s + timeout: 3s + retries: 12 + start_period: 10s + networks: + - thothii + + catalog-migrate: + image: thothii-core:local + profiles: [catalog-maintenance] + pull_policy: never + command: ["node", "/app/backend/dist/catalog/migrate.js"] + environment: + THT_CATALOG_DB_HOST: catalog-db + THT_CATALOG_DB_PORT: "5432" + THT_CATALOG_DB_NAME: thothii_catalog + THT_CATALOG_MIGRATOR_USER: thothii_catalog_migrate + THT_CATALOG_MIGRATOR_PASSWORD_FILE: /run/secrets/catalog_migrator_password + secrets: + - catalog_migrator_password + depends_on: + catalog-db: + condition: service_healthy + networks: + - thothii + workspace-maintenance: image: thothii-core:local profiles: [workspace-maintenance] @@ -209,7 +262,12 @@ volumes: qdrant-data: embedding-models: auth-state: + catalog-data: secrets: thothii_secrets: file: "${THT_SECRETS_FILE:?set THT_SECRETS_FILE}" + catalog_runtime_password: + file: "${THT_CATALOG_RUNTIME_PASSWORD_SOURCE:-./deploy/secrets/catalog-runtime-password}" + catalog_migrator_password: + file: "${THT_CATALOG_MIGRATOR_PASSWORD_SOURCE:-./deploy/secrets/catalog-migrator-password}" diff --git a/deploy/env/local.env.example b/deploy/env/local.env.example index d7627d2a..9961b32e 100644 --- a/deploy/env/local.env.example +++ b/deploy/env/local.env.example @@ -7,6 +7,9 @@ MAX_PI_PROCESSES=4 PI_AUTH_FILE=/absolute/path/to/pi-auth.json THT_SECRETS_FILE=/absolute/path/to/thothii.secrets THT_AUTH_CONFIG_ROOT=/absolute/path/to/thothii-auth +THT_CATALOG_RUNTIME_PASSWORD_SOURCE=/absolute/path/to/catalog-runtime-password +THT_CATALOG_MIGRATOR_PASSWORD_SOURCE=/absolute/path/to/catalog-migrator-password +THT_CATALOG_SYNC_TIMEOUT_MS=600000 THT_WORKSPACE_GIT_REMOTE=https://git.example.invalid/platform/thoth-workspaces.git THT_WORKSPACE_GIT_BRANCH=main diff --git a/deploy/env/server.env.example b/deploy/env/server.env.example index 152c3c2e..0c0211fe 100644 --- a/deploy/env/server.env.example +++ b/deploy/env/server.env.example @@ -6,6 +6,9 @@ MAX_PI_PROCESSES=4 PI_AUTH_FILE=/absolute/path/to/pi-auth.json THT_SECRETS_FILE=/absolute/path/to/thothii.secrets THT_AUTH_CONFIG_ROOT=/absolute/path/to/thothii-auth +THT_CATALOG_RUNTIME_PASSWORD_SOURCE=/absolute/path/to/catalog-runtime-password +THT_CATALOG_MIGRATOR_PASSWORD_SOURCE=/absolute/path/to/catalog-migrator-password +THT_CATALOG_SYNC_TIMEOUT_MS=600000 THT_DATA_ROOT=/srv/thothii/data THT_PI_STATE_ROOT=/srv/thothii/pi-state diff --git a/docker/catalog-db-init.sql b/docker/catalog-db-init.sql new file mode 100644 index 00000000..7fcb6ae8 --- /dev/null +++ b/docker/catalog-db-init.sql @@ -0,0 +1,25 @@ +DO $bootstrap$ +DECLARE + runtime_password text := trim(both E'\r\n' from pg_read_file('/run/secrets/catalog_runtime_password')); +BEGIN + IF runtime_password = '' THEN + RAISE EXCEPTION 'catalog runtime password is empty'; + END IF; + + IF NOT EXISTS ( + SELECT 1 FROM pg_catalog.pg_roles WHERE rolname = 'thothii_catalog_runtime' + ) THEN + EXECUTE format( + 'CREATE ROLE thothii_catalog_runtime LOGIN PASSWORD %L', + runtime_password + ); + END IF; +END +$bootstrap$; + +GRANT CONNECT ON DATABASE thothii_catalog TO thothii_catalog_runtime; +GRANT USAGE ON SCHEMA public TO thothii_catalog_runtime; +ALTER DEFAULT PRIVILEGES IN SCHEMA public + GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO thothii_catalog_runtime; +ALTER DEFAULT PRIVILEGES IN SCHEMA public + GRANT USAGE, SELECT ON SEQUENCES TO thothii_catalog_runtime; diff --git a/docker/core.Dockerfile b/docker/core.Dockerfile index 410eeff4..9a9c1cf6 100644 --- a/docker/core.Dockerfile +++ b/docker/core.Dockerfile @@ -96,7 +96,9 @@ RUN ln -s /opt/venv /app/harness/.venv # Backend: dist + node_modules (stesso Node major 24 + glibc bookworm → compatibili) COPY --from=backend-build /src/backend/dist /app/backend/dist COPY --from=backend-build /src/backend/node_modules /app/backend/node_modules +COPY backend/scripts/ssh-askpass.mjs /app/backend/scripts/ssh-askpass.mjs COPY backend/package*.json /app/backend/ +RUN chmod 0755 /app/backend/scripts/ssh-askpass.mjs # Runtime Pi is installed only from the committed lockfile. The image exposes its immutable # executable directly, so no host Pi installation or writable global npm directory is needed. diff --git a/docs/adr/0001-postgres-metadata-catalog.md b/docs/adr/0001-postgres-metadata-catalog.md index dfc857f9..675c461f 100644 --- a/docs/adr/0001-postgres-metadata-catalog.md +++ b/docs/adr/0001-postgres-metadata-catalog.md @@ -2,10 +2,12 @@ ThothII userà un Metadata Catalog PostgreSQL interno per conservare, per ogni workspace, la struttura fisica acquisita interrogando il relativo database e i metadati semantici generati con -l'AI. Ogni workspace avrà un solo Workspace Database; identità e lista dei workspace resteranno -autorevoli in `thoth-workspaces.yaml`, mentre il catalogo ne conserverà soltanto il riferimento -stabile. `schema/annotations.yaml` verrà sostituito come input del core in uno step successivo; -l'interfaccia e il lifecycle amministrativi resteranno separati dal workflow NL→SQL. +l'AI. Ogni Workspace Database conserverà un `workspace_id` obbligatorio e univoco: questo realizza +l'associazione uno-a-uno senza introdurre nel catalogo una tabella Workspace o una foreign key SQL. +Identità e lista dei workspace resteranno autorevoli in `thoth-workspaces.yaml`; il servizio +validerà il riferimento contro quel catalogo. `schema/annotations.yaml` verrà sostituito come input +del core in uno step successivo; l'interfaccia e il lifecycle amministrativi resteranno separati dal +workflow NL→SQL. ## Considered Options @@ -18,4 +20,7 @@ l'interfaccia e il lifecycle amministrativi resteranno separati dal workflow NL PSD importerà le annotations esistenti; gli altri workspace genereranno i metadati da zero. Il cutover futuro dovrà sostituire consapevolmente i consumatori delle annotations e verificarne -l'equivalenza semantica. Le sessioni di test esistenti non sono un vincolo di migrazione. +l'equivalenza semantica. PostgreSQL può garantire che uno stesso `workspace_id` non sia assegnato a +due database, ma l'esistenza del workspace e la gestione di rename o rimozioni restano responsabilità +del confine applicativo con il catalogo YAML. Le sessioni di test esistenti non sono un vincolo di +migrazione. diff --git a/docs/adr/0002-workspace-database-secret-references.md b/docs/adr/0002-workspace-database-secret-references.md new file mode 100644 index 00000000..550edaaf --- /dev/null +++ b/docs/adr/0002-workspace-database-secret-references.md @@ -0,0 +1,19 @@ +# Riferimenti al secret store per i Workspace Database + +Il Metadata Catalog non conserverà credenziali o chiavi dei Workspace Database. Riuserà il secret +store cifrato già posseduto da ThothII e conserverà soltanto riferimenti ai requisiti segreti del +workspace, evitando un secondo vault e impedendo che API, esportazioni o log espongano i valori. + +## Considered Options + +- Copiare i campi testuali del modello ThothAI avrebbe semplificato il CRUD, ma avrebbe conservato + password e passphrase in chiaro. +- Introdurre subito un secondo vault avrebbe separato il catalogo dal runtime workspace, ma avrebbe + duplicato cifratura, rotazione, autorizzazioni e procedure operative senza un'esigenza distinta. + +## Consequences + +Il catalogo e il runtime condividono l'identità dei requisiti segreti, mentre permessi e API del +catalogo restano separati. Sostituzione e cancellazione di un Workspace Database dovranno definire +esplicitamente il lifecycle dei relativi riferimenti senza includere i valori nelle transazioni del +catalogo PostgreSQL. diff --git a/docs/adr/0003-installation-local-database-bindings.md b/docs/adr/0003-installation-local-database-bindings.md new file mode 100644 index 00000000..caa91755 --- /dev/null +++ b/docs/adr/0003-installation-local-database-bindings.md @@ -0,0 +1,26 @@ +# Binding di database specifiche dell'installazione + +Il Metadata Catalog separa il Workspace Database logico dalla Database Binding che lo rende +raggiungibile in una specifica installazione. Esiste un solo Workspace Database per `workspace_id` +e una sola binding attiva nel catalogo di ciascuna installazione; per esempio PSD usa REST in locale +e PostgreSQL diretto sul server senza diventare due database distinti. + +Nel modello finale il catalogo è autorevole per engine, nome fisico, schema, capability e binding, +mentre `thoth-workspaces.yaml` conserva l'identità del workspace. Gli attuali campi DWH dei +descriptor sono una sorgente di bootstrap da importare e confrontare durante un cutover esplicito, +non una seconda fonte di verità permanente. + +## Considered Options + +- Conservare un record database per ogni trasporto avrebbe duplicato identità, struttura e + metadati dello stesso DWH fra locale e server. +- Conservare permanentemente i dati DWH sia nello YAML sia nel catalogo avrebbe introdotto + conflitti non risolvibili deterministicamente. +- Rendere globali le binding avrebbe mescolato endpoint e credenziali che appartengono a + installazioni con topologie e confini di sicurezza differenti. + +## Consequences + +La lista amministrativa unisce workspace YAML, database configurati e record orphaned. Il cutover +deve importare e confrontare la configurazione esistente prima di rimuoverla dai descriptor; il +runtime non deve osservare simultaneamente due autorità discordanti. diff --git a/docs/adr/0004-fastify-kysely-metadata-catalog.md b/docs/adr/0004-fastify-kysely-metadata-catalog.md new file mode 100644 index 00000000..bd317b83 --- /dev/null +++ b/docs/adr/0004-fastify-kysely-metadata-catalog.md @@ -0,0 +1,22 @@ +# Metadata Catalog nello stesso backend con Kysely + +Il Metadata Catalog vive nello stesso processo Fastify come modulo isolato, invece di introdurre un +microservizio. Usa il driver `pg` già presente attraverso Kysely per query, transazioni e migrazioni +tipizzate; route, repository, service, readiness e diagnostica restano separati dal workflow e +l'indisponibilità del catalogo non rende indisponibili sessioni o SSE. + +## Considered Options + +- Un microservizio avrebbe conservato letteralmente il backend bridge senza database, ma avrebbe + aggiunto deployment, autenticazione e failure mode per un solo contesto amministrativo. +- Usare soltanto `pg` avrebbe evitato una dipendenza, ma avrebbe richiesto infrastruttura locale per + transazioni, tipi delle righe, ordinamento e locking delle migrazioni. +- Drizzle o Prisma avrebbero aggiunto schema DSL, generatori e toolchain non necessari a un servizio + che vuole mantenere SQL e constraint PostgreSQL espliciti. + +## Consequences + +Il backend possiede una connection pool del catalogo e la chiude con il lifecycle Fastify. Le +migrazioni timestampate sono compilate insieme al backend ma vengono eseguite soltanto da +`catalog:migrate`, con credenziali migrator separate dal ruolo DML usato a runtime. Il modulo resta +dietro un'interfaccia repository per mantenere unit test e route test indipendenti da PostgreSQL. diff --git a/docs/adr/0005-hard-delete-catalog-tables-during-synchronization.md b/docs/adr/0005-hard-delete-catalog-tables-during-synchronization.md new file mode 100644 index 00000000..5918b3bd --- /dev/null +++ b/docs/adr/0005-hard-delete-catalog-tables-during-synchronization.md @@ -0,0 +1,27 @@ +# Hard-delete catalog tables during synchronization + +An explicit Table Synchronization makes the Catalog Table membership exactly match a successful +observation of the Workspace Database: new tables are created, source metadata is refreshed, and +absent tables plus their future column and relationship children are permanently deleted. Physical +membership cannot be edited manually. + +The external scan runs without holding a catalog transaction. Its diff is applied atomically only +while the Workspace Database version still matches the scanned binding. Failed scans change +nothing, and a non-empty removal set must exactly match the names confirmed by the operator; a +changed second scan therefore requires a new confirmation. + +## Considered Options + +- Soft deletion would preserve descriptions across accidental removals, but would add hidden state, + restore rules, and ambiguity about whether the catalog still represents the physical schema. +- Rename detection based on similarity would preserve metadata in some cases, but could silently + attach curated semantics to the wrong physical table. +- Append-only introspection, as in the legacy importer, would leave stale tables in the catalog and + make downstream schema linking unreliable. + +## Consequences + +A physical rename is delete plus create and loses curated metadata. The UI previews permanent +deletions, and future Catalog Column and Relationship records must cascade with their table. The +catalog remains an exact projection of the last accepted successful scan without tombstones or +restore lifecycle. diff --git a/docs/adr/0006-separate-physical-and-logical-relationships.md b/docs/adr/0006-separate-physical-and-logical-relationships.md new file mode 100644 index 00000000..36cb9774 --- /dev/null +++ b/docs/adr/0006-separate-physical-and-logical-relationships.md @@ -0,0 +1,8 @@ +# Separate physical and logical relationships + +ThothII persists each database-declared foreign-key constraint as an immutable Catalog +Relationship with ordered column pairs, so composite keys retain their identity and the database +remains the authority for physical structure. Curated or AI-inferred Logical Relationships will +use a separate future model and lifecycle rather than being mixed with physical constraints or +denormalized into textual column fields; this keeps synchronization authoritative without +preventing later semantic enrichment. diff --git a/docs/adr/0007-durable-authoritative-schema-synchronization.md b/docs/adr/0007-durable-authoritative-schema-synchronization.md new file mode 100644 index 00000000..6a5f855f --- /dev/null +++ b/docs/adr/0007-durable-authoritative-schema-synchronization.md @@ -0,0 +1,9 @@ +# Use durable runs for authoritative schema synchronization + +All table, column, relationship, and full-schema synchronizations run as durable background +Catalog Sync Runs rather than separate synchronous and asynchronous implementations. Each scope +is authoritative within its boundary, while Synchronize All observes one complete schema snapshot; +destructive diffs require confirmation and source revalidation before an atomic, fail-closed +catalog transaction. A persistent per-database lock, progress events, interruption handling, and +binding-version freshness make long operations observable and prevent two processes or stale +configuration from producing a partially trusted catalog. diff --git a/docs/agents/domain.md b/docs/agents/domain.md new file mode 100644 index 00000000..dead047c --- /dev/null +++ b/docs/agents/domain.md @@ -0,0 +1,7 @@ +# Domain Docs + +This is a single-context repository. + +Before exploring, read `CONTEXT.md` at the repository root and the relevant decisions in +`docs/adr/`. Use the project's terminology from `CONTEXT.md` in issue titles, proposals, and +tests. Surface conflicts with an ADR instead of silently overriding it. diff --git a/docs/agents/issue-tracker.md b/docs/agents/issue-tracker.md new file mode 100644 index 00000000..207d49ce --- /dev/null +++ b/docs/agents/issue-tracker.md @@ -0,0 +1,28 @@ +# Issue tracker: Gitea + +Issues and specs for this repository live in the self-hosted Gitea repository: +`https://git.tylconsulting.it/mptyl/ThothII`. + +## Conventions + +- **Create an issue**: use the repository's Gitea web UI, or the Gitea REST API when an + authenticated token with issue scope is available. +- **Read and list issues**: use the Gitea web UI or authenticated API; include labels and comments. +- **Apply or remove labels**: use the issue's label controls or the Gitea API. +- **Comment and close**: use the issue page or the Gitea API. +- Do not use `gh issue ...` for this repository: the `github` remote is a mirror, not the canonical + issue tracker. + +## Repository identity + +- Canonical Git remote: `origin` → `https://git.tylconsulting.it/mptyl/ThothII.git` +- GitHub mirror: `github` → `https://github.com/mptyl/ThothII.git` +- Canonical issue URL: `https://git.tylconsulting.it/mptyl/ThothII/issues` + +## When a skill says “publish to the issue tracker” + +Create an issue in the canonical Gitea repository. + +## When a skill says “fetch the relevant ticket” + +Read the referenced issue in the canonical Gitea repository. diff --git a/docs/agents/triage-labels.md b/docs/agents/triage-labels.md new file mode 100644 index 00000000..fbda9883 --- /dev/null +++ b/docs/agents/triage-labels.md @@ -0,0 +1,19 @@ +# Triage Labels + +The skills speak in terms of five canonical triage roles. This file maps those roles to the labels +used in the canonical Gitea issue tracker. + +| Label in mattpocock/skills | Label in Gitea | Meaning | +| --- | --- | --- | +| `needs-triage` | `needs-triage` | Maintainer needs to evaluate this issue | +| `needs-info` | `needs-info` | Waiting on reporter for more information | +| `ready-for-agent` | `ready-for-agent` | Fully specified, ready for AFK agent work | +| `ready-for-human` | `ready-for-human` | Requires human implementation | +| `wontfix` | `wontfix` | Will not be actioned | + +Issue type labels: + +| Label in Gitea | Meaning | +| --- | --- | +| `bug` | Something is not working | +| `enhancement` | New feature or request | diff --git a/docs/architecture/components.md b/docs/architecture/components.md index 878e66cc..3004bca3 100644 --- a/docs/architecture/components.md +++ b/docs/architecture/components.md @@ -4,7 +4,10 @@ This page complements the [architecture overview](overview.md) with the module s ## Modules and dependencies -The frontend communicates with the backend through REST and SSE. The backend does not own session persistence: it starts Pi, invokes the `tht` CLI, and forwards events. The harness contains the workflow, the Python CLI, and adapters for the DWH and vector store. +The frontend communicates with the backend through REST and SSE. The backend does not own session +persistence: it starts Pi, invokes the `tht` CLI, and forwards events. It does own the separate +installation-local database catalog. The harness contains the workflow, the Python CLI, and +adapters for the DWH and vector store. ```mermaid flowchart LR @@ -18,6 +21,8 @@ flowchart LR THT --> DWH["DWH\nread-only"] THT --> VDB["Qdrant / vector store"] BE --> CFG["settings.json\nworkspace registry"] + BE --> CAT["catalog-db\nPostgreSQL + Kysely"] + BE -->|catalog Test + Table Sync| DWH FE -.->|renders widgets| EXT ``` @@ -26,7 +31,7 @@ Dipendenze principali: | Module | Depends on | Responsibility | | --- | --- | --- | | `frontend/` | Backend REST and SSE APIs | UI, gate widgets, and in-memory transcript | -| `backend/src/` | Pi, `tht`, configuration, and workspace registry | Transport, session lifecycle, and APIs | +| `backend/src/` | Pi, `tht`, configuration, workspace registry, catalog PostgreSQL, and read-only DWH connectors | Transport, session lifecycle, catalog CRUD, connection tests, table introspection, and APIs | | `harness/.pi/` | Pi and `tht phase` | Workflow orchestration and human-in-the-loop gates | | `harness/tht/` | Filesystem, DWH, and vector store | Persistence, CLI, Evidence, schema, and preprocessing | | workspace repository | `source/`, `curated/`, manifest, and artifacts | Versioned Evidence source and session output | diff --git a/docs/architecture/overview.md b/docs/architecture/overview.md index 62123577..71899212 100644 --- a/docs/architecture/overview.md +++ b/docs/architecture/overview.md @@ -11,6 +11,7 @@ For sessions, roles, groups, diagnostics, and recovery, see the [authentication flowchart LR USER["Reviewer"] --> FE["Frontend\nReact and SSE"] FE --> BE["Backend\nFastify"] + BE --> CATALOG["Metadata catalog\nPostgreSQL"] BE --> PI["Pi\nRPC per sessione"] PI --> THT["tht and harness\nworkflow and persistence"] THT --> DWH["DWH\nread only"] @@ -27,8 +28,8 @@ frontend (React/SSE) → backend (Fastify) → pi --mode rpc → tht/harness → | Layer | Stack | Ruolo | |---|---|---| -| **harness/** | Python (`tht` CLI) + Pi gate extension (JS) | Owns the workflow and **all** persistence | -| **backend/** | Fastify + TypeScript | Thin bridge with no database of its own | +| **harness/** | Python (`tht` CLI) + Pi gate extension (JS) | Owns the workflow and all session persistence | +| **backend/** | Fastify + TypeScript + Kysely | Session bridge plus the isolated administrative metadata catalog | | **frontend/** | React 18 + Vite | UI that renders gate widgets and rebuilds the live transcript from the SSE stream | ## The harness owns the workflow @@ -39,14 +40,22 @@ frontend (React/SSE) → backend (Fastify) → pi --mode rpc → tht/harness → A session is a directory under `sessions/` (the workspace defines the path): `session_manifest.yaml`, phase artifacts (`question.md`, `schema_linking.json`, `sql_final.sql`, and others), and `review_decisions.jsonl`. The contract says: *"persisted state is the truth; what is not recorded did not happen"*. There is no verbatim transcript store. A resumed Pi process rebuilds context from `tht session show ` and the artifacts on disk. -## The backend is a thin bridge with no database +## The backend bridges sessions and owns the metadata catalog - `ThtRunner` runs `tht` subcommands in a shell. - `PiProcessManager` runs one Pi child process per session and bridges its RPC stream. - `SessionBridge` maps Pi RPC events to client events (`ui_request` / `text_delta` / `info`). - `SseHub` distributes these events to the browser over SSE. +- `CatalogService` joins authoritative YAML workspace identities with installation-local database + configurations stored in PostgreSQL through Kysely. +- `CatalogTableService` reconciles persisted Catalog Tables with a successful external schema scan; + `ConcreteCatalogTableIntrospector` isolates direct PostgreSQL, typed REST, and SSH-tunnel access. -Application settings live in a JSON file (`backend/data/settings.json`), not in a database. +Application settings remain in `backend/data/settings.json`; session state remains in harness phase +documents. PostgreSQL stores only the administrative database catalog, bindings, observed tables, +and curated descriptions. Connector secrets remain write-only in the encrypted workspace secret +store. Catalog SSH support is limited to connection tests and table synchronization; it does not +change the session runtime binding contract. ## Human-in-the-loop gate contract diff --git a/docs/contracts/catalog-schema-snapshot.md b/docs/contracts/catalog-schema-snapshot.md new file mode 100644 index 00000000..89336952 --- /dev/null +++ b/docs/contracts/catalog-schema-snapshot.md @@ -0,0 +1,88 @@ +# Catalog schema snapshot RPC + +Il percorso preferito per un binding `rest_api` espone al catalogo un'unica fotografia tipizzata +dello schema: + +```http +POST /rpc/schema_snapshot +Content-Type: application/json + +{"schema_name":"datawarehouse"} +``` + +La risposta è un oggetto JSON con `schemaVersion: 1`, capability esplicite e tre collezioni. Una +capability non disponibile deve essere dichiarata `unavailable`: non deve essere simulata con una +lista vuota. + +```json +{ + "schemaVersion": 1, + "capabilities": { + "tables": "available", + "columns": "available", + "relationships": "available" + }, + "tables": [ + { "name": "patients", "sourceComment": "Clinical patients" } + ], + "columns": [ + { + "tableName": "patients", + "name": "id", + "ordinalPosition": 1, + "dataType": "bigint", + "isNullable": false, + "defaultExpression": null, + "primaryKeyPosition": 1, + "sourceComment": "Patient identifier" + } + ], + "relationships": [ + { + "constraintName": "visits_patient_id_fkey", + "sourceTableName": "visits", + "targetTableName": "patients", + "updateRule": "NO ACTION", + "deleteRule": "CASCADE", + "deferrable": false, + "initiallyDeferred": false, + "columns": [ + { "position": 1, "sourceColumnName": "patient_id", "targetColumnName": "id" } + ] + } + ] +} +``` + +## Fallback compatibile tramite `run_query` + +Se e soltanto se il server non espone `POST /rpc/schema_snapshot`, il catalogo può ottenere la +stessa fotografia mediante una singola istruzione read-only inviata all'RPC già esistente: + +```http +POST /rpc/run_query +Content-Type: application/json + +{"query_text":"WITH ... SELECT ..."} +``` + +La query è costruita dal catalogo, interroga soltanto il catalogo PostgreSQL dello schema +configurato e aggrega tabelle, colonne, primary key e foreign key nella stessa istruzione. Non sono +ammessi più round trip, query per tabella o assemblaggi client-side di osservazioni effettuate in +momenti diversi. Il nome schema deve essere validato come identificatore e quotato come valore SQL, +non interpolato come SQL libero. + +`run_query` restituisce un array JSON di righe. Per questo fallback l'array deve contenere +esattamente una riga e quella riga deve essere esattamente l'oggetto snapshot v1 sopra descritto, +con `schemaVersion`, `capabilities`, `tables`, `columns` e `relationships`; campi mancanti, +aggiuntivi o di tipo diverso rendono invalida l'intera fotografia. La risposta non è un contratto +alternativo o più permissivo: cambia soltanto il trasporto della stessa snapshot stretta. + +`position` e `primaryKeyPosition` sono uno-based. Le coppie ordinate permettono foreign key +composte. Il catalogo rifiuta l'intera fotografia se il JSON non rispetta il contratto o se la +capability richiesta dal tipo di sincronizzazione è `unavailable`; in entrambi i casi non applica +alcuna modifica. Il fallback viene tentato soltanto quando l'RPC preferito risulta assente, non per +nascondere una snapshot malformata o un errore operativo del server. Se anche `run_query` non è +disponibile, la query viene rifiutata, la risposta non contiene una singola snapshot v1 valida o una +capability richiesta è `unavailable`, il run fallisce senza aggiornamenti parziali e senza esporre +il corpo remoto. diff --git a/docs/guida-utente.md b/docs/guida-utente.md index 95fe5072..7e2a54fa 100644 --- a/docs/guida-utente.md +++ b/docs/guida-utente.md @@ -92,8 +92,10 @@ evidence: Changes from older workspaces: -- the database is reached only through **REST** or **direct Postgres** (`rest_api` / - `postgres_direct`); the SSH tunnel remains disabled; +- NL→SQL sessions reach the database only through **REST** or **direct Postgres** (`rest_api` / + `postgres_direct`); `ssh_tunnel` remains disabled for session runtime. The separate Database + management surface supports SSH for **Test connection** and **Sync tables**, with a private key, + mandatory `known_hosts`, and an optional key passphrase; - the semantic index is **internal** (Qdrant plus `qwen3-embedding:0.6b`, 1024 dimensions, cosine); - **filesystem** Evidence lives in the repository (`/evidence`) and is materialized from the pinned Git commit (P6). HTTP Evidence is also supported. diff --git a/docs/plans/2026-08-26-metadata-catalog-from-thothai.md b/docs/plans/2026-08-26-metadata-catalog-from-thothai.md index 333c5640..461d4ada 100644 --- a/docs/plans/2026-08-26-metadata-catalog-from-thothai.md +++ b/docs/plans/2026-08-26-metadata-catalog-from-thothai.md @@ -1,8 +1,9 @@ # Metadata Catalog di ThothII: ricognizione ThothAI e percorso incrementale -Data: 2026-08-26 -Stato: ricognizione completata; step 1 implementato; scelte tecnologiche degli step successivi -deliberatamente rinviate. +Data: 2026-08-26; aggiornato 2026-08-27 +Stato: ricognizione e progettazione completate; navigazione, CRUD Workspace Database, Catalog +Table, Catalog Column, Catalog Relationship e sincronizzazione durevole dello schema implementati +il 2026-08-27. Generazione AI e integrazione con il workflow core restano negli step successivi. ## Obiettivo @@ -10,9 +11,11 @@ ThothII deve introdurre un contesto amministrativo separato, il **Metadata Catal il database associato a ciascun workspace, la sua struttura fisica introspezionata e i metadati semantici oggi rappresentati da `schema/annotations.yaml`. -Il programma procede per step indipendenti. Il primo step aggiunge soltanto l'accesso dalla sidebar -destra a una superficie centrale vuota. Non introduce PostgreSQL, API CRUD, introspezione o -integrazioni con il workflow core. +Il programma procede per step indipendenti. Il primo step ha aggiunto l'accesso dalla sidebar; il +secondo ha sostituito la superficie vuota con il CRUD di configurazione, il PostgreSQL interno e i +test di connessione; gli step successivi hanno aggiunto navigazione gerarchica, colonne, relazioni +fisiche e sincronizzazione durevole dell'intero schema. Non introduce ancora generazione AI o +integrazione con il workflow core. Questa analisi usa come riferimento il working tree legacy osservato in `Thoth/ThothAI`. Non è stato verificato che quel contenuto corrisponda a una release o a un tag @@ -20,8 +23,10 @@ canonico; i percorsi e i comportamenti descrivono il sorgente disponibile il 202 ## Decisioni già confermate -1. Ogni workspace è associato a un solo Workspace Database e ogni Workspace Database appartiene a - un solo workspace. +1. Ogni Workspace Database appartiene a un solo workspace tramite un `workspace_id` obbligatorio e + univoco; un workspace può avere al massimo un Workspace Database. Poiché i workspace non sono + righe del catalogo PostgreSQL, l'associazione è un riferimento logico validato contro + `thoth-workspaces.yaml`, non una foreign key SQL. 2. Il CRUD non crea né rinomina workspace. Identità e lista ordinata dei workspace restano autorevoli in `thoth-workspaces.yaml`; il catalogo conserva il loro identificatore stabile. 3. La struttura fisica viene acquisita interrogando il database esterno tramite i dati di @@ -38,6 +43,146 @@ canonico; i percorsi e i comportamenti descrivono il sorgente disponibile il 202 introduce un router. 9. La pagina iniziale è vuota, segue il tema, nasconde l'intera colonna core e non interrompe una sessione live. Le azioni di apertura, resume o creazione sessione riportano al core. +10. La compatibilità con il modello ThothAI è semantica, non una copia letterale: configurazione e + contenuti semantici sono campi relazionali mutabili, mentre identità e appartenenza della + struttura fisica derivano dall'introspezione; i segreti restano nel secret store e lo stato dei + job non viene mescolato ai dati amministrativi. +11. Il CRUD amministra il Metadata Catalog e non esegue DDL sul database esterno, che resta + read-only. +12. La prima versione supporta PostgreSQL; il confine di introspezione dovrà permettere di + aggiungere altri dialetti senza cambiare il modello del catalogo. +13. I segreti dei Workspace Database riusano il secret store cifrato di ThothII. Il catalogo + conserva riferimenti ai segreti e nessuna API, esportazione o log ne restituisce i valori. +14. La UI usa AG Grid Community per la lista master e un pannello React separato per il dettaglio; + non dipende dalle funzionalità master-detail di AG Grid Enterprise. +15. Un Workspace Database il cui `workspace_id` scompare dal catalogo YAML non viene cancellato + automaticamente: diventa orphaned e può soltanto essere recuperato, riassegnato o eliminato + esplicitamente da un amministratore. +16. La prima vertical slice gestisce configurazione del Workspace Database, riferimenti ai segreti, + test di connessione e stato. La seconda gestisce le Catalog Table: la collezione e i nomi sono + controllati dall'introspezione, mentre la descrizione curata è modificabile. Le slice successive + hanno aggiunto Catalog Column, Catalog Relationship e sincronizzazione durevole dello schema. +17. Il modello non conserva il `name` libero di ThothAI: nome e ID visualizzati appartengono al + workspace YAML, mentre `database_name` identifica il database PostgreSQL esterno. +18. Database management supporta i tre trasporti già riconosciuti da ThothII: `postgres_direct`, + `rest_api` e `ssh_tunnel`. PSD rimane un solo Workspace Database: usa la connessione diretta sul + server e l'endpoint REST in locale tramite una Database Binding specifica dell'installazione. + Questo supporto non abilita automaticamente `ssh_tunnel` nel runtime NL→SQL. +19. Una configurazione può essere salvata prima di una connessione riuscita. Il test separato + produce uno stato `untested`, `reachable` o `failed`; attivazione e introspezione richiedono uno + stato raggiungibile. +20. Il CRUD e il test di connessione richiedono `database.manage`; inserimento e sostituzione dei + segreti continuano a richiedere `workspace.secrets.manage`. +21. Il Workspace Database e il modo di raggiungerlo sono entità distinte. Ogni catalogo di + installazione conserva una sola Database Binding attiva per workspace: PSD usa `rest_api` in + locale e `postgres_direct` sul server senza duplicare il Workspace Database. +22. Nel modello finale il Metadata Catalog è autorevole per engine, `database_name`, schema, + capacità e binding. Lo YAML resta autorevole per identità e contenuti del workspace; i campi + DWH correnti saranno importati, confrontati e rimossi soltanto durante un cutover esplicito. +23. La lista master è l'unione fra workspace YAML e record del catalogo: mostra workspace + `unconfigured`, database configurati e record `orphaned`. +24. Ogni introspezione registra le capability disponibili. Una capability `unavailable` non viene + rappresentata come una collezione osservata ma vuota; REST può completare con successo anche + quando indici o enum non sono supportati. +25. Il Metadata Catalog non introduce snapshot, draft o pubblicazioni. Configurazione e contenuti + semantici, inclusi quelli futuri generati dall'AI, sono normali campi modificabili; la struttura + osservata cambia soltanto con una sincronizzazione esplicita. +26. Il normale Delete elimina realmente il Workspace Database, la Database Binding e i relativi + record catalogo e segreti. Non modifica il DWH esterno né il repository YAML; il workspace torna + visibile nella lista master come `unconfigured`. +27. La prima versione gestisce un solo schema obbligatorio per Workspace Database, identificato + dalla coppia `database_name + schema`; per PSD la coppia è `postgres + datawarehouse`. +28. I record mantengono soltanto `created_at`, `updated_at` e un contatore `version` per optimistic + concurrency. Non esistono storico delle revisioni, rollback o audit applicativo delle modifiche. +29. `workspace_databases` conserva soltanto UUID, `workspace_id` unique, engine, `database_name`, + schema, timestamp e version. Il nome visualizzato appartiene al workspace YAML. +30. Ogni Workspace Database ha al massimo una riga `database_bindings`. Una singola tabella usa + check constraint dipendenti da `transport` per i campi direct, REST e SSH; non esiste un flag + `active`, perché ciascuna installazione conserva una sola binding. +31. `rest_api` configura il Thoth REST Connector tipizzato: base URL, autenticazione e TLS sono dati + della binding, mentre path RPC e shape delle risposte appartengono al contratto applicativo e non + sono liberamente configurabili. +32. Il test connessione usa soltanto una configurazione già salvata ed è associato alla sua + `version`. Ogni modifica della binding o dei segreti invalida il risultato precedente e riporta + lo stato a `untested`. +33. Password, API key e chiavi sono write-only: l'API espone soltanto `configured`, un campo vuoto + conserva il valore esistente e la sostituzione è un'azione esplicita. Delete rimuove anche i + segreti associati. +34. La pagina usa AG Grid come master e un form React come detail, con sezioni Database, Connection + e TLS/SSH condizionali. La toolbar offre `Add database`; le righe `unconfigured` offrono + `Configure`. Entrambe selezionano esclusivamente workspace YAML senza un database e creano il + record soltanto al Save; `workspace_id` diventa immutabile dopo la creazione. +35. La grid mostra workspace, database, schema, transport, endpoint, stato connessione e ultimo + aggiornamento. Su schermi piccoli il dettaglio occupa il pannello completo. Il cambio riga con + modifiche non salvate e Delete richiedono conferma, senza conferma testuale tipizzata. +36. La Database Binding conserva `connection_status`, `tested_version`, `last_tested_at`, un codice + errore e un messaggio breve sanificato. Non conserva stack trace, DSN, credenziali o output grezzo + del driver. +37. Le API vivono sotto `/api/catalog`: list/create di `/databases`, get/patch/delete di + `/databases/:id`, sostituzione dei segreti sotto `/databases/:id/secrets`, test connessione sotto + `/databases/:id/test` e list/patch/sync delle tabelle sotto `/databases/:id/tables`. +38. `GET /api/catalog/databases` restituisce l'intera master list unificata; AG Grid Community applica + client-side ricerca, filtri e ordinamento. La prima versione non introduce paginazione server o + funzionalità AG Grid Enterprise. +39. Il Metadata Catalog vive nello stesso processo Fastify come modulo isolato con repository, + service, route, diagnostica e readiness proprie. L'indisponibilità del catalogo non modifica + sessioni, SSE o health del core e non giustifica ancora un microservizio separato. +40. Il backend mantiene `pg@8.22.0` e aggiunge `kysely@0.29.5` per query e transazioni tipizzate. Le + migrazioni Kysely sono timestampate, compilate con il backend ed eseguite da un comando + `catalog:migrate` separato; l'applicazione non migra automaticamente il database all'avvio. +41. Lo stack aggiunge un servizio interno `catalog-db` con volume persistente, ruolo runtime DML, + ruolo migrator DDL e job one-shot `catalog-migrate`. Un catalogo indisponibile produce 503 sulle + sole route catalogo. +42. La prima vertical slice è amministrativa: scrive il catalogo ma non cambia ancora il runtime di + sessioni e workflow, che continua a usare YAML e binding correnti fino al cutover esplicito. +43. `Configure` precompila senza salvare engine, database e schema dal descriptor e i dati non + sensibili dalla binding effettiva. L'amministratore verifica, inserisce i segreti e salva; non + esiste importazione silenziosa. +44. Unit e route test usano un repository fake; una suite PostgreSQL Testcontainers separata verifica + migrazioni, constraint, transazioni, optimistic concurrency e cascade. SQLite ed emulatori non + sono sostituti ammessi per questi test. +45. La navigazione delle entità catalogo è gerarchica e senza scorciatoie globali: `Databases → + Database → Overview | Tables → Table`. Non esistono una voce globale Tables, un filtro globale + Database o una preselezione implicita; Columns continuerà sotto Table e Relationships sotto + Database. +46. Una Catalog Table conserva nome fisico, `source_comment`, descrizione curata nullable, + `generated_description` nullable per lo step AI futuro, version e timestamp. La UI mostra come + tre campi indipendenti senza fallback visivo: source comment read-only, generated description + modificabile e description modificabile. I valori null restano celle e controlli vuoti. +47. Le Catalog Table non possono essere aggiunte, rinominate o cancellate manualmente. `Sync tables` + legge dal database esterno le tabelle PostgreSQL ordinarie e partizionate dello schema scelto; + viste e materialized view sono escluse. +48. La sincronizzazione è esplicita. La scansione avviene fuori dalla transazione del catalogo; il + diff viene applicato atomicamente soltanto se la version del Workspace Database è ancora quella + sottoposta a scansione. Una scansione fallita non modifica il catalogo. +49. Tabelle nuove vengono create, i commenti sorgente vengono aggiornati e quelle non più osservate + vengono eliminate definitivamente. La rimozione di tabelle, colonne o relazioni richiede la + conferma dell'esatto piano distruttivo; se il secondo scan produce una fotografia differente, + l'applicazione richiede una nuova conferma. +50. Un rename fisico è intenzionalmente delete più create e perde i metadati curati. Le colonne e + relazioni dipendenti vengono eliminate in cascade insieme alla Catalog Table. +51. L'introspezione vive nel modulo catalogo Fastify dietro un adapter. PostgreSQL diretto e tunnel + SSH usano il catalogo `pg_catalog`; REST preferisce il contratto tipizzato + `POST /rpc/schema_snapshot` e, quando quell'RPC non è esposto, usa come fallback compatibile una + singola query read-only tramite `POST /rpc/run_query`. Entrambi i percorsi devono produrre la + stessa fotografia v1 stretta descritta in `docs/contracts/catalog-schema-snapshot.md`. +52. Test connessione e sincronizzazione sono serializzati per Workspace Database, hanno timeout e + richiedono che la binding nella version corrente abbia un test `reachable` prima di qualsiasi + Catalog Sync Run. La scansione asincrona ha un timeout separato, di default dieci minuti. +53. Il tunnel SSH usa OpenSSH in modalità stdio `-W`, chiave privata e passphrase opzionale dal + secret store, `known_hosts` obbligatorio, `StrictHostKeyChecking=yes`, agent e configurazione + globale disabilitati. Non è ammesso TOFU. TLS PostgreSQL con CA e server name resta verificato + anche attraverso il tunnel. +54. In questo slice `ssh_tunnel` è una binding supportata da Database management per Test connection + e Schema Sync. Il renderer e il runtime delle sessioni NL→SQL restano fuori scope e continuano a + rifiutarla finché non verrà deciso il relativo cutover. +55. I menu di azione a livello Workspace Database espongono separatamente `Synchronize tables`, + `Synchronize all columns`, `Synchronize relationships` e `Synchronize all`. Su una selezione di + database lo scope scelto viene avviato per ogni database idoneo; non viene sostituito + implicitamente con una sincronizzazione completa. +56. Per lo scope Columns, `tableIds` vuoto significa tutte le Catalog Table correnti del Workspace + Database; `tableIds` valorizzato limita invece la riconciliazione alle tabelle indicate. La grid + Tables espone `Synchronize columns` sulle tabelle selezionate. ## Correzione del modello mentale corrente @@ -205,8 +350,8 @@ template Django. Il comportamento è principalmente additivo: usa `get_or_create` o controlli `exists`, aggiorna alcuni commenti, ma non riconcilia in modo completo rename, rimozioni o drift. Non va copiato così -com'è. Il futuro processo ThothII dovrà almeno distinguere scansione, differenze osservate e -applicazione della nuova snapshot. +com'è. Il processo ThothII implementato distingue scansione, differenze osservate e applicazione +della nuova snapshot. ### Generazione AI legacy @@ -318,7 +463,8 @@ rendering, retrieval, LSH o SQL generation. ### Vincoli minimi da progettare -- `workspace_id` unico sul Workspace Database; +- `workspace_id` obbligatorio e unico sul Workspace Database, con esistenza validata contro il + catalogo YAML dal servizio applicativo; - nome tabella unico nel database e schema appropriato; - nome colonna unico nella tabella; - relationship unica secondo il modello, anche per chiavi composite; @@ -348,8 +494,7 @@ L'export legacy della struttura include username e password in chiaro. Il modell password, passphrase SSH e altri segreti in `CharField`; non è stata trovata cifratura applicativa, nonostante un testo admin affermi il contrario. -Per ThothII resta da decidere nello step infrastrutturale quali dati di connessione siano normali -metadati e quali siano secret reference. In ogni caso: +ThothII distingue i metadati di connessione dai riferimenti al secret store cifrato. In ogni caso: - nessun endpoint o export deve restituire segreti; - log ed errori devono sanificare DSN e credenziali; @@ -357,6 +502,11 @@ metadati e quali siano secret reference. In ogni caso: - il catalogo non deve riusare credenziali del DWH, delle sessioni o di Qdrant; - test connessione e introspezione devono usare timeout e privilegi read-only. +La binding REST corrente richiede una verifica prima del cutover: il renderer emette +`ssl_ca_file`, mentre il modello Python espone `ssl_ca`; il percorso della CA privata potrebbe quindi +non essere consumato. PSD richiede TLS con CA privata in locale, perciò questo disallineamento deve +essere corretto e coperto da un test end-to-end prima di affidare il profilo REST al catalogo. + ## Percorso incrementale ### Step 1: accesso alla superficie vuota @@ -384,15 +534,17 @@ npx tsc -b ### Step 2: contratto di dominio e schema relazionale -Da progettare con un nuovo round decisionale: campi, secret reference, dialetti supportati, -namespace/schema, snapshot fisiche, relazioni fisiche/logiche e lifecycle dell'output AI. Nessuna -tecnologia ORM o migration tool è stata scelta in questo documento. +Progettazione della vertical slice completata: Workspace Database, Database Binding, singolo schema, +riferimenti al secret store, optimistic concurrency e capability per trasporto hanno contratti +espliciti. Configurazione e contenuti semantici restano mutabili; la struttura fisica osservata è +sincronizzata e non modificabile manualmente. ### Step 3: PostgreSQL interno e migrazioni -Da progettare separatamente dal core: servizio, volume, ruoli runtime/migrator/backup, health e -readiness dedicati, backup/restore e diagnostica. La sua indisponibilità non dovrà cambiare -`core /health` o interrompere una sessione. +PostgreSQL interno con volume e ruoli runtime/migrator separati. Il modulo catalogo usa Kysely sopra +il driver `pg`; le migrazioni compilate vengono applicate soltanto dal comando `catalog:migrate` e +mai allo startup Fastify. Health, readiness e diagnostica restano dedicate; l'indisponibilità del +catalogo non cambia `core /health` e non interrompe una sessione. ### Step 4: API CRUD @@ -401,20 +553,56 @@ Gli endpoint dovranno vivere sotto un namespace catalogo e non riutilizzare le r ### Step 5: UI CRUD -Liste e form per Workspace Database, tabelle, colonne e relazioni, costruiti con React/Vite e il -design system ThothII. La gerarchia e i filtri ThothAI sono il riferimento funzionale; Django Admin -non è il riferimento tecnologico o visuale. +Workspace Database, Catalog Table, Catalog Column e Catalog Relationship sono implementati con +React/Vite e il design system ThothII. +La navigazione è gerarchica e locale al database (`Overview | Tables`), senza menu o filtri globali +per tipo di entità. La grid delle tabelle non offre Add/Delete; il dettaglio full-width mantiene +immutabili i fatti fisici e consente di modificare separatamente Description e Generated +Description. Colonne e relazioni seguono la stessa gerarchia: Columns appartiene al dettaglio +della tabella, Relationships al database. I valori descrittivi null sono mostrati come celle e +campi vuoti, senza fallback visivi o placeholder `Not set` che nascondano quale sorgente è +effettivamente valorizzata. + +Le griglie che dispongono di azioni massive usano checkbox e una toolbar contestuale con conteggio, +menu `Actions` e cancellazione della selezione. La selezione identifica ID espliciti, può essere +accumulata attraverso i filtri e viene azzerata dopo successo, nuova sincronizzazione o uscita +dalla pagina; un'azione è all-or-nothing se un elemento non è idoneo. I menu a livello database +espongono gli scope fisici come azioni distinte: `Synchronize tables`, `Synchronize all columns`, +`Synchronize relationships` e `Synchronize all`. La grid Tables espone invece `Synchronize +columns` per le tabelle selezionate. Test connection resta un'azione distinta; griglie senza azioni +non mostrano controlli di selezione inerti. ### Step 6: introspezione -Connessione read-only, preview delle differenze, acquisizione di una Physical Schema Snapshot, -policy per rename/rimozioni e stato del job. Nessuna chiamata lunga dovrà mantenere aperta una -transazione CRUD. +Catalog Table, Catalog Column e Catalog Relationship sono implementate per PostgreSQL diretto, +Thoth REST Connector e tunnel SSH. La scansione read-only è separata dalla transazione; una +riconciliazione atomica crea, aggiorna i commenti sorgente ed elimina, dopo conferma, i fatti fisici +assenti senza rendere modificabile manualmente la struttura osservata. Gli scope autorevoli sono +Tables per database e Physical Relationships per database. Per Columns, `tableIds` vuoto include +tutte le Catalog Table correnti, mentre una lista di ID limita lo scope al sottoinsieme esplicito; +`Synchronize all` osserva tutti e tre gli scope in un unico snapshot e li riconcilia insieme. Tutti +gli scope sono eseguiti come Catalog Sync Run durevoli in background, non attraverso implementazioni +sincrone e asincrone separate. Un run che prevede cancellazioni conserva il diff, attende una +conferma esplicita e verifica nuovamente lo snapshot prima dell'applicazione; se la sorgente è +cambiata, invalida la conferma. Ogni applicazione è atomica e fail-closed: errori, timeout o +capability non disponibili non producono aggiornamenti parziali. + +PK e FK devono essere visibili sulle Catalog Column senza duplicare le stringhe denormalizzate di +ThothAI. La posizione nella primary key è un fatto osservato della colonna; membership e conteggio +FK sono proiezioni derivate dalle Catalog Relationship e dalle loro coppie ordinate, aggiornate +nella stessa transazione di riconciliazione. + +Ogni scope registra la versione della Database Binding osservata e l'istante dell'ultima +sincronizzazione. Una modifica della binding conserva il catalogo precedente ma lo marca stale; +solo un `Synchronize all` riuscito rende nuovamente corrente l'intero schema. ### Step 7: generazione AI dei metadati -Generazione di descrizioni e altri campi equivalenti alle annotations, editing umano e gestione -esplicita di errori o output non validi. Approvazione/versioning saranno decisi in questo step. +Generated Description è una proposta distinta e modificabile: un revisore può correggerla prima +di consolidarla esplicitamente come Description. Lo slice AI dovrà decidere e implementare anche +alias semantici, descrizioni dei valori, sinonimi e concetti per tabelle e colonne, oltre alla +gestione esplicita di errori e output non validi. La generazione AI e l'azione di consolidamento non +appartengono allo slice di introspezione dello schema. ### Step 8: migrazione PSD @@ -427,11 +615,24 @@ ricevono import legacy. Rimuovere la dipendenza da `annotations.yaml` soltanto dopo avere un contratto equivalente, test di rendering/search/Qdrant e una policy di disponibilità. Le sessioni di test esistenti possono essere eliminate, ma le nuove sessioni non devono osservare aggiornamenti parziali. +Questo cutover è esplicitamente rinviato fino al completamento del database dei metadati. Il primo +gate successivo obbligatorio sarà valutare l'integrazione del Catalog Schema Snapshot con il +workflow core e lo schema-linking corrente; il rinvio non autorizza a dimenticare o assorbire +implicitamente il lavoro in altri slice. ### Step 10: operazioni e accettazione -Backup/restore reale, diagnostica, metriche, audit, permessi definitivi, hardening degli export e -test di failure isolation fra catalogo e workflow. +Backup/restore reale, diagnostica, metriche, permessi definitivi, hardening degli export e +test di failure isolation fra catalogo e workflow. I Catalog Sync Run hanno un solo job attivo per +Workspace Database, sono concorrenti fra database diversi e usano un lock persistente. Un pannello +operativo non modale rimane visibile durante la navigazione del database, mostra fasi, contatori, +tempo trascorso e log sanitizzato via SSE con polling di fallback, e offre Confirm, Cancel e Retry +quando consentiti. Un restart marca `interrupted` i run rimasti attivi; il retry crea un nuovo run. +Le modifiche ai metadati restano consentite durante la scansione e sono preservate dall'applicazione. +Il worker gira inizialmente nello stesso servizio Fastify ma dietro un'interfaccia estraibile, con +coda, lease e heartbeat persistiti nel catalog-db. I riepiloghi dei run non scadono; gli eventi +dettagliati sono conservati per 30 giorni, mentre snapshot e diff completi vengono eliminati dopo +la conclusione lasciando conteggi, decisioni e una sintesi sanitizzata dell'esito. ## Verifiche del core da conservare per il cutover @@ -476,15 +677,11 @@ Definiscono gli effetti semantici e le guardie da mantenere o sostituire consape Le seguenti scelte non appartengono allo step 1: -- framework del servizio catalogo e libreria di accesso PostgreSQL; -- collocazione e protezione delle credenziali dei Workspace Database; -- supporto iniziale di dialetti diversi da PostgreSQL; -- uno o più schema namespace per database; -- policy di reconciliation per rename e delete; -- modello delle foreign key composite; -- distinzione persistente fra relationship fisiche e logiche; -- lifecycle draft/review/approval dell'output AI; -- versionamento, audit e rollback; +- lifecycle dei riferimenti ai segreti durante sostituzione e cancellazione; +- criteri per aggiungere dialetti successivi a PostgreSQL; +- criteri per un'eventuale estensione futura a più schemi per database; +- lifecycle e gestione amministrativa delle future Logical Relationship; +- alias semantici, descrizioni dei valori, sinonimi e concetti prodotti o assistiti dall'AI; - formato e momento del cutover dal file al database interno; - permission definitiva separata da `workspace.manage`. diff --git a/frontend/src/api/catalog-databases.ts b/frontend/src/api/catalog-databases.ts new file mode 100644 index 00000000..04a56df7 --- /dev/null +++ b/frontend/src/api/catalog-databases.ts @@ -0,0 +1,265 @@ +import { apiFetch, assertSameOriginRequestUrl, BASE } from "./client"; +import { joinBackendPath } from "./runtime-config"; + +export type DatabaseTransport = "postgres_direct" | "rest_api" | "ssh_tunnel"; +export type ConnectionStatus = "untested" | "reachable" | "failed"; +export type CatalogSecretName = + | "password" + | "apiKey" + | "sshPrivateKey" + | "sshPrivateKeyPassphrase" + | "sshKnownHosts" + | "tlsCa"; + +export interface DatabaseBinding { + transport: DatabaseTransport; + host?: string; + port?: number; + username?: string; + baseUrl?: string; + restPath?: string; + restAuth?: "none" | "bearer" | "x-api-key"; + tlsServername?: string; + sshHost?: string; + sshPort?: number; + sshUsername?: string; + sshTargetHost?: string; + sshTargetPort?: number; +} + +export interface CatalogDatabase { + id?: string; + workspaceId: string; + workspaceName: string; + workspaceDescription?: string; + workspaceAvailable: boolean; + configured: boolean; + engine: "postgres"; + databaseName: string; + schema: string; + version: number; + createdAt: string; + updatedAt: string; + binding: DatabaseBinding; + connectionStatus: ConnectionStatus; + testedVersion?: number; + lastTestedAt?: string; + lastErrorCode?: string; + lastErrorMessage?: string; + schemaSyncedVersion?: number; + schemaSyncedAt?: string; + activeSyncRun?: CatalogSyncRun; + secrets: Record; +} + +export interface DatabaseConfiguration { + workspaceId: string; + engine: "postgres"; + databaseName: string; + schema: string; + binding: DatabaseBinding; +} + +export interface CatalogTable { + id: string; + databaseId: string; + name: string; + sourceComment: string | null; + description: string | null; + generatedDescription: string | null; + lastSyncedDatabaseVersion?: number | null; + lastSyncedAt?: string | null; + version: number; + createdAt: string; + updatedAt: string; +} + +export interface CatalogColumn { + id: string; + tableId: string; + name: string; + ordinalPosition: number; + dataType: string; + isNullable: boolean; + defaultExpression: string | null; + primaryKeyPosition: number | null; + isPrimaryKey: boolean; + isForeignKey: boolean; + foreignKeyCount: number; + sourceComment: string | null; + description: string | null; + generatedDescription: string | null; + lastSyncedDatabaseVersion: number | null; + lastSyncedAt: string | null; + version: number; + createdAt: string; + updatedAt: string; +} + +export interface CatalogRelationshipColumn { + position: number; + sourceColumnId: string; + sourceColumnName: string; + targetColumnId: string; + targetColumnName: string; +} + +export interface CatalogRelationship { + id: string; + databaseId: string; + constraintName: string; + sourceTableId: string; + sourceTableName: string; + targetTableId: string; + targetTableName: string; + updateRule: string; + deleteRule: string; + deferrable: boolean; + initiallyDeferred: boolean; + columns: CatalogRelationshipColumn[]; + lastSyncedDatabaseVersion: number | null; + lastSyncedAt: string | null; + createdAt: string; + updatedAt: string; +} + +export type CatalogSyncScope = "tables" | "columns" | "relationships" | "all"; +export type CatalogSyncState = "queued" | "running" | "awaiting_confirmation" | "applying" + | "succeeded" | "failed" | "cancelled" | "interrupted"; +export type CatalogSyncPhase = "queued" | "connecting" | "scanning_tables" | "scanning_columns" + | "scanning_relationships" | "planning" | "awaiting_confirmation" | "applying" | "completed"; + +export interface CatalogSchemaDiff { + deletedTables: string[]; + deletedColumns: Array<{ tableName: string; columnName: string }>; + deletedRelationships: Array<{ sourceTableName: string; constraintName: string }>; +} + +export interface CatalogSyncRun { + id: string; + databaseId: string; + scope: CatalogSyncScope; + tableIds: string[]; + state: CatalogSyncState; + phase: CatalogSyncPhase; + requestedDatabaseVersion: number; + plannedDiff: CatalogSchemaDiff | null; + confirmationToken: string | null; + counts: { tables?: number; columns?: number; relationships?: number; created?: number; updated?: number; deleted?: number }; + errorCode: string | null; + errorMessage: string | null; + cancelRequested: boolean; + createdAt: string; + startedAt: string | null; + updatedAt: string; + finishedAt: string | null; + heartbeatAt: string | null; +} + +export interface CatalogSyncEvent { + id: number; + runId: string; + sequence: number; + level: "info" | "warning" | "error"; + eventType: string; + message: string; + data: Record; + createdAt: string; +} + +export const listCatalogDatabases = () => apiFetch("/catalog/databases"); + +export const createCatalogDatabase = (input: DatabaseConfiguration) => + apiFetch("/catalog/databases", { method: "POST", body: JSON.stringify(input) }); + +export const updateCatalogDatabase = (id: string, version: number, input: DatabaseConfiguration) => + apiFetch(`/catalog/databases/${encodeURIComponent(id)}`, { + method: "PATCH", + body: JSON.stringify({ ...input, version }), + }); + +export const replaceCatalogDatabaseSecrets = ( + id: string, + version: number, + values: Partial>, +) => apiFetch(`/catalog/databases/${encodeURIComponent(id)}/secrets`, { + method: "PUT", + body: JSON.stringify({ version, values }), +}); + +export const testCatalogDatabase = (id: string, version: number) => + apiFetch(`/catalog/databases/${encodeURIComponent(id)}/test`, { + method: "POST", + body: JSON.stringify({ version }), + }); + +export const deleteCatalogDatabase = (id: string, version: number) => + apiFetch(`/catalog/databases/${encodeURIComponent(id)}?version=${version}`, { method: "DELETE" }); + +export const listCatalogTables = (databaseId: string) => + apiFetch(`/catalog/databases/${encodeURIComponent(databaseId)}/tables`); + +export const updateCatalogTableMetadata = ( + databaseId: string, + tableId: string, + version: number, + description: string | null, + generatedDescription: string | null, +) => apiFetch( + `/catalog/databases/${encodeURIComponent(databaseId)}/tables/${encodeURIComponent(tableId)}`, + { method: "PATCH", body: JSON.stringify({ version, description, generatedDescription }) }, +); + +export const listCatalogColumns = (databaseId: string, tableId: string) => + apiFetch(`/catalog/databases/${encodeURIComponent(databaseId)}/tables/${encodeURIComponent(tableId)}/columns`); + +export const updateCatalogColumnMetadata = ( + databaseId: string, + tableId: string, + columnId: string, + version: number, + description: string | null, + generatedDescription: string | null, +) => apiFetch( + `/catalog/databases/${encodeURIComponent(databaseId)}/tables/${encodeURIComponent(tableId)}/columns/${encodeURIComponent(columnId)}`, + { method: "PATCH", body: JSON.stringify({ version, description, generatedDescription }) }, +); + +export const listCatalogRelationships = (databaseId: string) => + apiFetch(`/catalog/databases/${encodeURIComponent(databaseId)}/relationships`); + +export const startCatalogSync = ( + databaseId: string, + version: number, + scope: CatalogSyncScope, + tableIds: string[] = [], +) => apiFetch(`/catalog/databases/${encodeURIComponent(databaseId)}/sync-runs`, { + method: "POST", + body: JSON.stringify({ version, scope, tableIds }), +}); + +export const listCatalogSyncRuns = (databaseId: string) => + apiFetch(`/catalog/databases/${encodeURIComponent(databaseId)}/sync-runs`); + +export const getCatalogSyncRun = (runId: string) => + apiFetch(`/catalog/sync-runs/${encodeURIComponent(runId)}`); + +export const listCatalogSyncEvents = (runId: string, after = 0) => + apiFetch(`/catalog/sync-runs/${encodeURIComponent(runId)}/events-list?after=${after}`); + +export const confirmCatalogSync = (runId: string, confirmationToken: string) => + apiFetch(`/catalog/sync-runs/${encodeURIComponent(runId)}/confirm`, { + method: "POST", body: JSON.stringify({ confirmationToken }), + }); + +export const cancelCatalogSync = (runId: string) => + apiFetch(`/catalog/sync-runs/${encodeURIComponent(runId)}/cancel`, { method: "POST" }); + +export const retryCatalogSync = (runId: string) => + apiFetch(`/catalog/sync-runs/${encodeURIComponent(runId)}/retry`, { method: "POST" }); + +export function catalogSyncEventsUrl(runId: string, after = 0): string { + const url = joinBackendPath(BASE, `/catalog/sync-runs/${encodeURIComponent(runId)}/events?after=${after}`); + assertSameOriginRequestUrl(url); + return url; +} diff --git a/frontend/src/api/client.ts b/frontend/src/api/client.ts index ec92d02b..4822e094 100644 --- a/frontend/src/api/client.ts +++ b/frontend/src/api/client.ts @@ -14,6 +14,10 @@ const safeErrorCodes = new Set([ "git_unavailable", "git_auth_failed", "git_non_fast_forward", "connector_unavailable", "semantic_index_incompatible", "pi_management_forbidden", "pi_management_unavailable", "pi_management_invalid_config", "pi_management_write_failed", + "catalog_unavailable", "database_conflict", "database_invalid", "database_not_found", + "database_stale", "database_operation_failed", + "schema_sync_conflict", "schema_introspection_failed", "schema_request_invalid", + "schema_operation_failed", "sync_run_not_found", "table_stale", "column_stale", ]); type SafeErrorPayload = { @@ -46,6 +50,19 @@ const localCodeMessages: Record = { pi_management_unavailable: "Pi management is unavailable.", pi_management_invalid_config: "The Pi configuration is invalid.", pi_management_write_failed: "The Pi configuration could not be saved.", + catalog_unavailable: "The database catalog is unavailable.", + database_conflict: "This workspace already has a database configuration.", + database_invalid: "The database configuration is invalid.", + database_not_found: "The database configuration was not found.", + database_stale: "The database configuration changed. Reload and try again.", + database_operation_failed: "The database operation failed.", + schema_sync_conflict: "A schema synchronization is already active or no longer current.", + schema_introspection_failed: "The database schema could not be read safely.", + schema_request_invalid: "The schema request is invalid.", + schema_operation_failed: "The schema operation failed.", + sync_run_not_found: "The synchronization run was not found.", + table_stale: "Table metadata changed. Reload and try again.", + column_stale: "Column metadata changed. Reload and try again.", }; const localStatusMessages: Record = { diff --git a/frontend/src/index.css b/frontend/src/index.css index 2ac59958..86bad42c 100644 --- a/frontend/src/index.css +++ b/frontend/src/index.css @@ -296,3 +296,52 @@ 0%, 100% { transform: scale(1); box-shadow: 0 0 0 0 oklch(var(--primary) / 0.4); } 50% { transform: scale(1.15); box-shadow: 0 0 0 5px oklch(var(--primary) / 0); } } + +@layer components { + .thot-database-grid { + --ag-font-family: var(--font-sans); + --ag-font-size: 0.8125rem; + --ag-background-color: oklch(var(--card)); + --ag-foreground-color: oklch(var(--foreground)); + --ag-header-background-color: oklch(var(--muted) / 0.72); + --ag-header-foreground-color: oklch(var(--foreground)); + --ag-border-color: oklch(var(--border)); + --ag-row-border-color: oklch(var(--border) / 0.72); + --ag-odd-row-background-color: oklch(var(--muted) / 0.18); + --ag-row-hover-color: oklch(var(--muted) / 0.55); + --ag-selected-row-background-color: oklch(var(--primary) / 0.07); + --ag-input-focus-border-color: oklch(var(--primary) / 0.6); + --ag-range-selection-border-color: oklch(var(--primary) / 0.6); + --ag-header-column-separator-color: oklch(var(--border)); + --ag-header-column-separator-display: block; + --ag-wrapper-border-radius: 0; + --ag-cell-horizontal-padding: 12px; + } + .thot-database-grid .ag-root-wrapper { + border: 0; + } + .thot-database-grid .ag-cell { + display: flex; + align-items: center; + } + .thot-database-grid .ag-header-cell-label { + font-weight: 700; + } + .thot-database-grid .ag-cell-focus:not(.ag-cell-range-selected) { + outline: 2px solid oklch(var(--ring) / 0.45); + outline-offset: -2px; + } + .thot-database-grid .ag-cell.thot-database-actions-cell { + padding-inline: 2px; + } + .thot-database-grid .ag-cell.thot-database-actions-cell.ag-cell-focus:not(.ag-cell-range-selected) { + outline: none; + } + .thot-database-grid .ag-cell.thot-database-status-cell { + padding-inline: 4px; + } + .thot-database-grid .ag-pinned-right-header, + .thot-database-grid .ag-pinned-right-cols-container { + box-shadow: -1px 0 0 oklch(var(--border)); + } +} diff --git a/frontend/src/shell/AppShell.database-management.test.tsx b/frontend/src/shell/AppShell.database-management.test.tsx index 84eb3786..cbe7e056 100644 --- a/frontend/src/shell/AppShell.database-management.test.tsx +++ b/frontend/src/shell/AppShell.database-management.test.tsx @@ -23,7 +23,7 @@ beforeEach(() => { issuer: "test", subject: "test", roles: ["admin"], - permissions: ["session.use", "workspace.manage", "workspace.secrets.manage", "pi.manage"], + permissions: ["session.use", "workspace.manage", "workspace.secrets.manage", "database.manage", "pi.manage"], isAdmin: true, csrfToken: null, session: null, @@ -52,7 +52,7 @@ beforeEach(() => { ); }); -test("opens the blank database management surface and returns to the core", async () => { +test("opens the database management surface and returns to the core", async () => { renderShell(); const composer = screen.getByRole("textbox", { name: /new question/i }); @@ -122,7 +122,7 @@ test("keeps a live core session connected and returns when that session is opene expect(FakeEventSource.instances).toHaveLength(1); }); -test("hides all management entries from non-admin users", () => { +test("keeps read-safe workspace access but hides privileged management entries", () => { clearAuthState(); setAuthState({ issuer: "test", @@ -136,7 +136,52 @@ test("hides all management entries from non-admin users", () => { renderShell(); - expect(screen.queryByRole("button", { name: "Workspace management" })).not.toBeInTheDocument(); + expect(screen.getByRole("button", { name: "Workspace management" })).toBeInTheDocument(); expect(screen.queryByRole("button", { name: "Database management" })).not.toBeInTheDocument(); expect(screen.queryByRole("button", { name: "Pi management" })).not.toBeInTheDocument(); }); + +test("does not leave database management without confirming a dirty form", async () => { + server.use(http.get("/api/catalog/databases", () => HttpResponse.json([{ + id: "11111111-1111-4111-8111-111111111111", + workspaceId: "psd-clinical", + workspaceName: "Policlinico San Donato", + workspaceAvailable: true, + configured: true, + engine: "postgres", + databaseName: "warehouse", + schema: "datawarehouse", + version: 3, + createdAt: "2026-08-27T08:00:00Z", + updatedAt: "2026-08-27T09:00:00Z", + binding: { transport: "postgres_direct", host: "db.internal", port: 5432, username: "reader" }, + connectionStatus: "untested", + secrets: { + password: false, + apiKey: false, + sshPrivateKey: false, + sshPrivateKeyPassphrase: false, + sshKnownHosts: false, + tlsCa: false, + }, + }]))); + const confirm = vi.spyOn(window, "confirm").mockReturnValue(false); + renderShell(); + + await userEvent.click(screen.getByRole("button", { name: "Database management" })); + await userEvent.click(await screen.findByRole("button", { name: "Edit Policlinico San Donato" })); + const schema = screen.getByLabelText("Schema"); + await userEvent.clear(schema); + await userEvent.type(schema, "reporting"); + + await userEvent.click(screen.getByRole("button", { name: "New session" })); + + expect(confirm).toHaveBeenCalledWith("Discard unsaved database changes and leave database management?"); + expect(screen.getByRole("main", { name: "Database management" })).toBeVisible(); + expect(schema).toHaveValue("reporting"); + + confirm.mockReturnValue(true); + await userEvent.click(screen.getByRole("button", { name: "New session" })); + await waitFor(() => expect(screen.queryByRole("main", { name: "Database management" })).not.toBeInTheDocument()); + confirm.mockRestore(); +}); diff --git a/frontend/src/shell/AppShell.tsx b/frontend/src/shell/AppShell.tsx index ba3644ff..fefe489a 100644 --- a/frontend/src/shell/AppShell.tsx +++ b/frontend/src/shell/AppShell.tsx @@ -32,7 +32,7 @@ import { import type { SessionScope, SessionSummary } from "../api/types"; import { useAuthGeneration, useAuthUser } from "../auth/authState"; import { useQuery, useQueryClient } from "@tanstack/react-query"; -import { useEffect, useMemo, useRef, useState } from "react"; +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import type { CSSProperties } from "react"; import { captureAuthOperation, isAuthOperationCurrent, StaleAuthOperationError, type AuthOperationGuard } from "../auth/authOperation"; @@ -104,8 +104,8 @@ export function AppShell({ canLogout }: AppShellProps) { const canReadAllSessions = permissions.includes("session.read_all"); const canManageWorkspace = permissions.includes("workspace.manage"); const canManageWorkspaceSecrets = permissions.includes("workspace.secrets.manage"); + const canManageDatabase = permissions.includes("database.manage"); const canManagePi = permissions.includes("pi.manage"); - const isAdmin = authenticatedUser?.isAdmin === true; const authGeneration = useAuthGeneration(); const { data: sessions = [] } = useQuery({ queryKey: ["sessions", sessionScope], @@ -123,6 +123,10 @@ export function AppShell({ canLogout }: AppShellProps) { const queryClient = useQueryClient(); const [showActivity, setShowActivity] = useState(false); const [activeSurface, setActiveSurface] = useState("core"); + const databaseNavigationRef = useRef({ dirty: false, busy: false }); + const updateDatabaseNavigationState = useCallback((state: { dirty: boolean; busy: boolean }) => { + databaseNavigationRef.current = state; + }, []); const [workspaceManagerOpen, setWorkspaceManagerOpen] = useState(false); const [piManagementOpen, setPiManagementOpen] = useState(false); const [activeOpen, setActiveOpen] = useState(true); @@ -197,7 +201,20 @@ export function AppShell({ canLogout }: AppShellProps) { setSelectedSessionIds(selected ? new Set(sessions.map((session) => session.id)) : new Set()); } + function canLeaveDatabaseManagement(): boolean { + if (activeSurface !== "database-management") return true; + if (databaseNavigationRef.current.busy) { + toast.info("Wait for the database operation to finish before leaving this page"); + return false; + } + if (databaseNavigationRef.current.dirty) { + return window.confirm("Discard unsaved database changes and leave database management?"); + } + return true; + } + function openPanel(id: string) { + if (!canLeaveDatabaseManagement()) return; const s = sessions.find((x) => x.id === id); if (!s) return; setActiveSurface("core"); @@ -207,7 +224,7 @@ export function AppShell({ canLogout }: AppShellProps) { // completed sessions keep the read-only documents panel (with its explicit Resume), // so a mere click never spawns a runtime. if (s.active && s.status === "open" && !s.archived && !isForeignSession(s)) { - void doResume(id); + void doResume(id, true); return; } setPanelSession(s); setShowActivity(false); @@ -219,7 +236,8 @@ export function AppShell({ canLogout }: AppShellProps) { return next; }); } - async function doResume(id: string) { + async function doResume(id: string, databaseExitApproved = false) { + if (!databaseExitApproved && !canLeaveDatabaseManagement()) return; setActiveSurface("core"); const guard = captureAuthOperation({ sessionId: id, disposalEpoch: operationEpochRef.current }); if (!guard) return; @@ -502,6 +520,7 @@ export function AppShell({ canLogout }: AppShellProps) { }, [lastSystemEvent]); function startNewSession() { + if (!canLeaveDatabaseManagement()) return; setActiveSurface("core"); invalidateResumeIntent(); newSessionOperationRef.current = null; @@ -566,6 +585,7 @@ export function AppShell({ canLogout }: AppShellProps) { } async function signOut() { + if (!canLeaveDatabaseManagement()) return; await logoutUser(); } @@ -619,7 +639,13 @@ export function AppShell({ canLogout }: AppShellProps) { {/* Conversation column */}
- {activeSurface === "database-management" && } + {activeSurface === "database-management" && ( + + )}
New session - {isAdmin && ( - <> - + onClick={() => { + if (!canLeaveDatabaseManagement()) return; + setActiveSurface("core"); + setWorkspaceManagerOpen(true); + }} + > + Workspace management + + {canManageDatabase && ( - )} - {isAdmin && canManagePi && ( + {canManagePi && ( diff --git a/frontend/src/shell/DatabaseManagementPage.test.tsx b/frontend/src/shell/DatabaseManagementPage.test.tsx new file mode 100644 index 00000000..9aba9f91 --- /dev/null +++ b/frontend/src/shell/DatabaseManagementPage.test.tsx @@ -0,0 +1,842 @@ +import { act, render, screen, waitFor, within } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { http, HttpResponse } from "msw"; +import { server } from "../test/msw"; +import type { CatalogColumn, CatalogDatabase, CatalogSyncRun, CatalogTable } from "../api/catalog-databases"; +import { DatabaseManagementPage } from "./DatabaseManagementPage"; + +const noSecrets = { + password: false, + apiKey: false, + sshPrivateKey: false, + sshPrivateKeyPassphrase: false, + sshKnownHosts: false, + tlsCa: false, +}; + +function makeDatabase(overrides: Partial = {}): CatalogDatabase { + return { + id: "11111111-1111-4111-8111-111111111111", + workspaceId: "psd-clinical", + workspaceName: "Policlinico San Donato", + workspaceAvailable: true, + configured: true, + engine: "postgres", + databaseName: "warehouse", + schema: "datawarehouse", + version: 3, + createdAt: "2026-08-27T08:00:00Z", + updatedAt: "2026-08-27T09:00:00Z", + binding: { + transport: "postgres_direct", + host: "db.internal", + port: 5432, + username: "reader", + }, + connectionStatus: "untested", + secrets: { ...noSecrets }, + ...overrides, + }; +} + +const unconfigured = makeDatabase({ + id: undefined, + workspaceId: "lab", + workspaceName: "Research laboratory", + configured: false, + databaseName: "lab_warehouse", + schema: "analytics", + version: 0, + createdAt: "", + updatedAt: "", + binding: { transport: "postgres_direct", port: 5432 }, +}); + +const orphan = makeDatabase({ + id: "22222222-2222-4222-8222-222222222222", + workspaceId: "retired", + workspaceName: "Retired workspace", + workspaceAvailable: false, +}); + +function renderPage({ + rows = [makeDatabase(), unconfigured, orphan], + canManage = true, + canManageSecrets = true, + onNavigationStateChange, +}: { + rows?: CatalogDatabase[] | (() => CatalogDatabase[]); + canManage?: boolean; + canManageSecrets?: boolean; + onNavigationStateChange?: (state: { dirty: boolean; busy: boolean }) => void; +} = {}) { + server.use(http.get("/api/catalog/databases", () => HttpResponse.json( + typeof rows === "function" ? rows() : rows, + ))); + const client = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + const view = render( + + + , + ); + return { ...view, client }; +} + +function makeSyncRun( + scope: CatalogSyncRun["scope"], + tableIds: string[] = [], +): CatalogSyncRun { + return { + id: `sync-${scope}`, + databaseId: "11111111-1111-4111-8111-111111111111", + scope, + tableIds, + state: "queued", + phase: "queued", + requestedDatabaseVersion: 3, + plannedDiff: null, + confirmationToken: null, + counts: {}, + errorCode: null, + errorMessage: null, + cancelRequested: false, + createdAt: "2026-08-27T10:00:00Z", + startedAt: null, + updatedAt: "2026-08-27T10:00:00Z", + finishedAt: null, + heartbeatAt: null, + }; +} + +function registerCompletedSyncRun(run: CatalogSyncRun) { + const completed: CatalogSyncRun = { + ...run, + state: "succeeded", + phase: "completed", + startedAt: "2026-08-27T10:00:00Z", + finishedAt: "2026-08-27T10:00:01Z", + }; + server.use( + http.get("/api/catalog/sync-runs/:runId", () => HttpResponse.json(completed)), + http.get("/api/catalog/sync-runs/:runId/events-list", () => HttpResponse.json([])), + http.get("/api/catalog/databases/:databaseId/sync-runs", () => HttpResponse.json([completed])), + ); +} + +const synchronizationScopes = [ + { scope: "tables", label: "Synchronize tables" }, + { scope: "columns", label: "Synchronize all columns" }, + { scope: "relationships", label: "Synchronize relationships" }, + { scope: "all", label: "Synchronize all" }, +] as const; + +test("starts with a full-width list and applies the row action matrix", async () => { + renderPage(); + + expect(await screen.findByRole("button", { name: "View Policlinico San Donato" })).toBeEnabled(); + expect(screen.getByRole("button", { name: "Edit Policlinico San Donato" })).toBeEnabled(); + expect(screen.getByRole("button", { name: "Delete Policlinico San Donato" })).toBeEnabled(); + + expect(screen.getByRole("button", { name: "View Research laboratory" })).toBeEnabled(); + expect(screen.getByRole("button", { name: "Edit Research laboratory" })).toBeEnabled(); + expect(screen.getByRole("button", { name: "Delete Research laboratory" })).toBeDisabled(); + + expect(screen.getByRole("button", { name: "View Retired workspace" })).toBeEnabled(); + expect(screen.getByRole("button", { name: "Edit Retired workspace" })).toBeDisabled(); + expect(screen.getByRole("button", { name: "Delete Retired workspace" })).toBeEnabled(); + expect(screen.queryByRole("heading", { name: "Database details" })).not.toBeInTheDocument(); +}); + +test.each(synchronizationScopes)( + "selected database Actions offers and starts $scope synchronization", + async ({ scope, label }) => { + const user = userEvent.setup(); + let startBody: unknown; + const run = makeSyncRun(scope); + registerCompletedSyncRun(run); + server.use(http.post("/api/catalog/databases/:databaseId/sync-runs", async ({ request }) => { + startBody = await request.json(); + return HttpResponse.json(run, { status: 202 }); + })); + renderPage({ + rows: [makeDatabase({ connectionStatus: "reachable", testedVersion: 3 })], + }); + + const databaseRow = await screen.findByRole("row", { name: /Policlinico San Donato/ }); + await user.click(within(databaseRow).getByRole("checkbox")); + await user.click(screen.getByRole("button", { name: "Actions" })); + + expect(await screen.findByRole("menuitem", { name: synchronizationScopes[0].label })).toBeEnabled(); + for (const option of synchronizationScopes.slice(1)) { + expect(screen.getByRole("menuitem", { name: option.label })).toBeEnabled(); + } + await user.click(screen.getByRole("menuitem", { name: label })); + + await waitFor(() => expect(startBody).toEqual({ version: 3, scope, tableIds: [] })); + }, +); + +test("presents completed synchronization steps as success and skips unneeded confirmation", async () => { + const user = userEvent.setup(); + const run = { + ...makeSyncRun("columns"), + counts: { tables: 163, columns: 2_275 }, + }; + registerCompletedSyncRun(run); + server.use(http.post("/api/catalog/databases/:databaseId/sync-runs", () => ( + HttpResponse.json(run, { status: 202 }) + ))); + renderPage({ + rows: [makeDatabase({ connectionStatus: "reachable", testedVersion: 3 })], + }); + + const databaseRow = await screen.findByRole("row", { name: /Policlinico San Donato/ }); + await user.click(within(databaseRow).getByRole("checkbox")); + await user.click(screen.getByRole("button", { name: "Actions" })); + await user.click(await screen.findByRole("menuitem", { name: "Synchronize all columns" })); + + expect(await screen.findByRole("heading", { name: "Succeeded" })).toBeVisible(); + const steps = screen.getByRole("list", { name: "Synchronization steps" }); + expect(steps).toHaveClass("grid-cols-2", "sm:grid-cols-3"); + const connected = within(steps).getByRole("listitem", { name: "Connected, completed" }); + expect(connected.querySelector("svg")).toHaveClass("text-[oklch(var(--success))]"); + expect(within(steps).getByText("Tables read")).toBeVisible(); + const confirmation = within(steps).getByRole("listitem", { + name: "Confirmation not required, not required", + }); + expect(confirmation.querySelector("svg")).toHaveClass("text-muted-foreground"); + const completed = within(steps).getByRole("listitem", { name: "Completed, completed" }); + expect(completed.querySelector("svg")).toHaveClass("text-[oklch(var(--success))]"); + expect(steps.querySelectorAll(".text-primary")).toHaveLength(0); +}); + +test("database Overview exposes every synchronization scope", async () => { + const user = userEvent.setup(); + let startBody: unknown; + const run = makeSyncRun("relationships"); + registerCompletedSyncRun(run); + server.use(http.post("/api/catalog/databases/:databaseId/sync-runs", async ({ request }) => { + startBody = await request.json(); + return HttpResponse.json(run, { status: 202 }); + })); + renderPage({ + rows: [makeDatabase({ connectionStatus: "reachable", testedVersion: 3 })], + }); + + await user.click(await screen.findByRole("button", { name: "View Policlinico San Donato" })); + await user.click(screen.getByRole("button", { name: "Synchronize database schema" })); + + expect(await screen.findByRole("menuitem", { name: synchronizationScopes[0].label })).toBeEnabled(); + for (const option of synchronizationScopes.slice(1)) { + expect(screen.getByRole("menuitem", { name: option.label })).toBeEnabled(); + } + await user.click(screen.getByRole("menuitem", { name: "Synchronize relationships" })); + + await waitFor(() => expect(startBody).toEqual({ + version: 3, + scope: "relationships", + tableIds: [], + })); +}); + +test("replaces the list with View and returns focus to the originating action", async () => { + const user = userEvent.setup(); + renderPage({ rows: [makeDatabase()] }); + const view = await screen.findByRole("button", { name: "View Policlinico San Donato" }); + + await user.click(view); + + expect(screen.getByRole("heading", { name: "Database details" })).toBeVisible(); + expect(screen.getByDisplayValue("db.internal")).toHaveAttribute("readonly"); + expect(screen.getByRole("button", { name: "Test connection" })).toBeEnabled(); + expect(screen.queryByRole("button", { name: "Save changes" })).not.toBeInTheDocument(); + + await user.click(screen.getByRole("button", { name: "Back to list" })); + + await waitFor(() => expect(view).toHaveFocus()); + expect(screen.getByRole("button", { name: "View Policlinico San Donato" })).toBeVisible(); +}); + +test("opens an unconfigured row as Edit while creating its first saved configuration", async () => { + const user = userEvent.setup(); + let postBody: unknown; + const saved = makeDatabase({ + ...unconfigured, + id: "33333333-3333-4333-8333-333333333333", + configured: true, + version: 1, + binding: { transport: "rest_api", baseUrl: "https://psd.example/api", restPath: "/health", restAuth: "x-api-key" }, + }); + let rows = [unconfigured]; + server.use( + http.post("/api/catalog/databases", async ({ request }) => { + postBody = await request.json(); + rows = [saved]; + return HttpResponse.json(saved, { status: 201 }); + }), + ); + renderPage({ rows: () => rows }); + + await user.click(await screen.findByRole("button", { name: "Edit Research laboratory" })); + + expect(screen.getByRole("heading", { name: "Edit database" })).toBeVisible(); + expect(screen.getByLabelText("Workspace")).toBeDisabled(); + await user.selectOptions(screen.getByLabelText("Transport"), "rest_api"); + await user.type(screen.getByLabelText("Base URL"), "https://psd.example/api"); + expect(screen.getByLabelText("Diagnostic endpoint")).toHaveValue("/health"); + expect(screen.getByLabelText("Diagnostic endpoint")).toHaveAttribute("readonly"); + + await user.click(screen.getByRole("button", { name: "Save database" })); + + await waitFor(() => expect(postBody).toMatchObject({ + workspaceId: "lab", + binding: expect.objectContaining({ transport: "rest_api", baseUrl: "https://psd.example/api" }), + })); + expect(await screen.findByRole("heading", { name: "Edit database" })).toBeVisible(); +}); + +test("global Add offers only unconfigured workspaces and lets the operator choose one", async () => { + const user = userEvent.setup(); + const second = makeDatabase({ + ...unconfigured, + workspaceId: "radiology", + workspaceName: "Radiology", + databaseName: "radiology_dwh", + }); + renderPage({ rows: [makeDatabase(), unconfigured, second] }); + + await user.click(await screen.findByRole("button", { name: "Add database" })); + + expect(screen.getByRole("heading", { name: "Add database" })).toBeVisible(); + expect(screen.getByRole("button", { name: "Add database" })).toBeVisible(); + const selector = screen.getByLabelText("Workspace"); + expect(selector).toBeEnabled(); + expect(screen.queryByRole("option", { name: "Policlinico San Donato" })).not.toBeInTheDocument(); + await user.selectOptions(selector, "radiology"); + expect(screen.getByDisplayValue("radiology_dwh")).toBeVisible(); +}); + +test("guards a dirty Edit, disables testing, and reports navigation state", async () => { + const user = userEvent.setup(); + const navigation = vi.fn(); + const confirm = vi.spyOn(window, "confirm").mockReturnValue(false); + renderPage({ rows: [makeDatabase()], onNavigationStateChange: navigation }); + + await user.click(await screen.findByRole("button", { name: "Edit Policlinico San Donato" })); + const schema = screen.getByLabelText("Schema"); + await user.clear(schema); + await user.type(schema, "reporting"); + + expect(screen.getByRole("button", { name: "Test connection" })).toBeDisabled(); + await waitFor(() => expect(navigation).toHaveBeenLastCalledWith({ dirty: true, busy: false })); + await user.click(screen.getByRole("button", { name: "Back to list" })); + expect(confirm).toHaveBeenCalledWith("Discard unsaved database changes?"); + expect(screen.getByRole("heading", { name: "Edit database" })).toBeVisible(); + + confirm.mockReturnValue(true); + await user.click(screen.getByRole("button", { name: "Back to list" })); + expect(await screen.findByRole("button", { name: "Edit Policlinico San Donato" })).toBeVisible(); +}); + +test("uses an in-page destructive form and returns the YAML workspace to Not configured", async () => { + const user = userEvent.setup(); + let row = makeDatabase(); + let deleteCalled = false; + let deleteVersion: string | null = null; + server.use( + http.delete("/api/catalog/databases/:id", ({ request }) => { + deleteCalled = true; + deleteVersion = new URL(request.url).searchParams.get("version"); + row = { + ...row, + id: undefined, + configured: false, + version: 0, + updatedAt: "", + connectionStatus: "untested", + secrets: { ...noSecrets }, + }; + return new HttpResponse(null, { status: 204 }); + }), + ); + renderPage({ rows: () => [row] }); + + await user.click(await screen.findByRole("button", { name: "Delete Policlinico San Donato" })); + expect(screen.getByRole("heading", { name: "Delete database" })).toBeVisible(); + expect(screen.getByText("This removes the local database configuration.")).toBeVisible(); + expect(screen.queryByRole("dialog")).not.toBeInTheDocument(); + + await user.click(screen.getByRole("button", { name: "Delete database" })); + + await waitFor(() => expect(deleteCalled).toBe(true)); + expect(deleteVersion).toBe("3"); + expect(await screen.findByRole("button", { name: "Delete Policlinico San Donato" })).toBeDisabled(); + expect(screen.getByText("Not configured")).toBeVisible(); +}); + +test("tests only the persisted version and updates the visible connection status", async () => { + const user = userEvent.setup(); + let testBody: unknown; + let row = makeDatabase(); + server.use(http.post("/api/catalog/databases/:id/test", async ({ request }) => { + testBody = await request.json(); + row = makeDatabase({ version: 4, connectionStatus: "reachable", testedVersion: 4 }); + return HttpResponse.json(row); + })); + renderPage({ rows: () => [row] }); + + await user.click(await screen.findByRole("button", { name: "View Policlinico San Donato" })); + await user.click(screen.getByRole("button", { name: "Test connection" })); + + await waitFor(() => expect(testBody).toEqual({ version: 3 })); + expect(within(screen.getByRole("region", { name: "Database details form" })).getByText("Reachable")).toBeVisible(); +}); + +test("retains a stale draft and requires an explicit reload", async () => { + const user = userEvent.setup(); + server.use( + http.patch("/api/catalog/databases/:id", () => HttpResponse.json({ + code: "database_stale", + message: "The database changed", + }, { status: 409 })), + ); + renderPage({ rows: [makeDatabase()] }); + + await user.click(await screen.findByRole("button", { name: "Edit Policlinico San Donato" })); + const schema = screen.getByLabelText("Schema"); + await user.clear(schema); + await user.type(schema, "draft_schema"); + await user.click(screen.getByRole("button", { name: "Save changes" })); + + expect(await screen.findByText("A newer database configuration is available.")).toBeVisible(); + expect(schema).toHaveValue("draft_schema"); + expect(screen.getByRole("button", { name: "Save changes" })).toBeDisabled(); + + await user.click(screen.getByRole("button", { name: "Keep editing" })); + expect(screen.getByText("Reload the latest values before this configuration can be changed.")).toBeVisible(); + expect(schema).toHaveValue("draft_schema"); +}); + +test("marks an open form stale when a background refetch advances the catalog version", async () => { + const user = userEvent.setup(); + const { client } = renderPage({ rows: [makeDatabase()] }); + + await user.click(await screen.findByRole("button", { name: "Edit Policlinico San Donato" })); + act(() => { + client.setQueryData(["catalog-databases"], [makeDatabase({ version: 4, schema: "server_schema" })]); + }); + + expect(await screen.findByText("A newer database configuration is available.")).toBeVisible(); + expect(screen.getByLabelText("Schema")).toHaveValue("datawarehouse"); + expect(screen.getByRole("button", { name: "Save changes" })).toBeDisabled(); +}); + +test("does not submit credentials hidden by a transport or authentication change", async () => { + const user = userEvent.setup(); + let putCalled = false; + let patchBody: Record | undefined; + let row = makeDatabase(); + server.use( + http.patch("/api/catalog/databases/:id", async ({ request }) => { + patchBody = await request.json() as Record; + row = makeDatabase({ + version: 4, + binding: { transport: "rest_api", baseUrl: "https://psd.example/api", restPath: "/health", restAuth: "none" }, + }); + return HttpResponse.json(row); + }), + http.put("/api/catalog/databases/:id/secrets", () => { + putCalled = true; + return HttpResponse.json(row); + }), + ); + renderPage({ rows: () => [row] }); + + await user.click(await screen.findByRole("button", { name: "Edit Policlinico San Donato" })); + await user.type(screen.getByLabelText("Password"), "must-not-be-sent"); + await user.selectOptions(screen.getByLabelText("Transport"), "rest_api"); + await user.type(screen.getByLabelText("Base URL"), "https://psd.example/api"); + await user.selectOptions(screen.getByLabelText("Authentication"), "none"); + await user.click(screen.getByRole("button", { name: "Save changes" })); + + await waitFor(() => expect(patchBody).toBeDefined()); + expect(putCalled).toBe(false); + expect(patchBody).toMatchObject({ + version: 3, + binding: { + transport: "rest_api", + baseUrl: "https://psd.example/api", + restPath: "/health", + restAuth: "none", + }, + }); + expect((patchBody?.binding as Record).host).toBeUndefined(); +}); + +test("preserves typed secrets and offers a retry when the configuration save is only partial", async () => { + const user = userEvent.setup(); + let putCalls = 0; + let row = makeDatabase(); + server.use( + http.put("/api/catalog/databases/:id/secrets", async ({ request }) => { + putCalls += 1; + const body = await request.json(); + expect(body).toEqual({ version: 3, values: { password: "transient-secret" } }); + if (putCalls === 1) { + return HttpResponse.json({ + code: "catalog_unavailable", + message: "Secret store unavailable", + }, { status: 503 }); + } + row = makeDatabase({ version: 4, secrets: { ...noSecrets, password: true } }); + return HttpResponse.json(row); + }), + ); + renderPage({ rows: () => [row] }); + + await user.click(await screen.findByRole("button", { name: "Edit Policlinico San Donato" })); + await user.type(screen.getByLabelText("Password"), "transient-secret"); + await user.click(screen.getByRole("button", { name: "Save changes" })); + + expect(await screen.findByText("Database configuration saved; secret update could not be confirmed.")).toBeVisible(); + expect(screen.getByLabelText("Password")).toHaveValue("transient-secret"); + expect(screen.getByRole("button", { name: "Retry secrets" })).toBeEnabled(); + + await user.clear(screen.getByLabelText("Schema")); + await user.type(screen.getByLabelText("Schema"), "reporting_after_retry"); + await user.click(screen.getByRole("button", { name: "Retry secrets" })); + + await waitFor(() => expect(putCalls).toBe(2)); + expect(screen.queryByText("Database configuration saved; secret update could not be confirmed.")).not.toBeInTheDocument(); + expect(screen.getByLabelText("Schema")).toHaveValue("reporting_after_retry"); + expect(screen.getByRole("button", { name: "Save changes" })).toBeEnabled(); +}); + +test("shows secret status without exposing or enabling values when permission is absent", async () => { + const user = userEvent.setup(); + renderPage({ + rows: [makeDatabase({ secrets: { ...noSecrets, password: true } })], + canManageSecrets: false, + }); + + await user.click(await screen.findByRole("button", { name: "View Policlinico San Donato" })); + expect(screen.getByLabelText("Password")).toHaveValue("Configured"); + expect(screen.getByLabelText("Password")).toHaveAttribute("readonly"); + + await user.click(screen.getByRole("button", { name: "Back to list" })); + await user.click(screen.getByRole("button", { name: "Edit Policlinico San Donato" })); + expect(await screen.findByRole("heading", { name: "Edit database" })).toBeVisible(); + expect(screen.getByLabelText("Password")).toBeDisabled(); + expect(screen.getByText(/requires the workspace\.secrets\.manage permission/i)).toBeVisible(); +}); + +const patientsTable: CatalogTable = { + id: "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa", + databaseId: "11111111-1111-4111-8111-111111111111", + name: "patients", + sourceComment: "Patients imported from the clinical source", + description: null, + generatedDescription: null, + version: 1, + createdAt: "2026-08-27T08:00:00Z", + updatedAt: "2026-08-27T09:00:00Z", +}; + +test("filters catalog tables as the operator types", async () => { + const user = userEvent.setup(); + const mediciTable: CatalogTable = { + ...patientsTable, + id: "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb", + name: "bridge_medici", + sourceComment: "Doctors participating in clinical care", + }; + server.use( + http.get("/api/catalog/databases/:databaseId/tables", () => HttpResponse.json([ + patientsTable, + mediciTable, + ])), + ); + renderPage({ + rows: [makeDatabase({ connectionStatus: "reachable", testedVersion: 3 })], + }); + + await user.click(await screen.findByRole("button", { name: "View Policlinico San Donato" })); + await user.click(screen.getByRole("tab", { name: "Tables" })); + const tablesRegion = await screen.findByRole("region", { name: "Tables for Policlinico San Donato" }); + expect(await screen.findByRole("button", { name: "Edit description for patients" })).toBeVisible(); + expect(screen.getByRole("button", { name: "Edit description for bridge_medici" })).toBeVisible(); + + await user.type(screen.getByRole("textbox", { name: "Search tables" }), "medici"); + + expect(screen.getByRole("button", { name: "Edit description for bridge_medici" })).toBeVisible(); + expect(await within(tablesRegion).findByText("1 of 2")).toBeVisible(); + await waitFor(() => { + expect(screen.queryByRole("button", { name: "Edit description for patients" })).not.toBeInTheDocument(); + }); +}); + +test("selected table exposes a direct Synchronize columns action and sends its id", async () => { + const user = userEvent.setup(); + let startBody: unknown; + const run = makeSyncRun("columns", [patientsTable.id]); + registerCompletedSyncRun(run); + server.use( + http.get("/api/catalog/databases/:databaseId/tables", () => HttpResponse.json([patientsTable])), + http.post("/api/catalog/databases/:databaseId/sync-runs", async ({ request }) => { + startBody = await request.json(); + return HttpResponse.json(run, { status: 202 }); + }), + ); + renderPage({ + rows: [makeDatabase({ connectionStatus: "reachable", testedVersion: 3 })], + }); + + await user.click(await screen.findByRole("button", { name: "View Policlinico San Donato" })); + await user.click(screen.getByRole("tab", { name: "Tables" })); + const tableRow = await screen.findByRole("row", { name: /patients/ }); + await user.click(within(tableRow).getByRole("checkbox")); + + const synchronizeColumns = screen.getByRole("button", { name: "Synchronize columns" }); + expect(synchronizeColumns).toBeVisible(); + expect(synchronizeColumns).toBeEnabled(); + expect(screen.queryByRole("button", { name: "Actions" })).not.toBeInTheDocument(); + await user.click(synchronizeColumns); + + await waitFor(() => expect(startBody).toEqual({ + version: 3, + scope: "columns", + tableIds: [patientsTable.id], + })); + await waitFor(() => expect(screen.queryByText("1 selected")).not.toBeInTheDocument()); +}); + +test("navigates purely from a database to its tables and edits review metadata", async () => { + const user = userEvent.setup(); + let patchBody: unknown; + server.use( + http.get("/api/catalog/databases/:databaseId/tables", () => HttpResponse.json([patientsTable])), + http.patch("/api/catalog/databases/:databaseId/tables/:tableId", async ({ request }) => { + patchBody = await request.json(); + return HttpResponse.json({ + ...patientsTable, + description: "Registry used for longitudinal patient analysis", + version: 2, + }); + }), + ); + renderPage({ + rows: [makeDatabase({ connectionStatus: "reachable", testedVersion: 3 })], + }); + + await user.click(await screen.findByRole("button", { name: "View Policlinico San Donato" })); + await user.click(screen.getByRole("tab", { name: "Tables" })); + + expect(await screen.findByRole("region", { name: "Tables for Policlinico San Donato" })).toBeVisible(); + expect(screen.getByRole("tab", { name: "Tables" })).toHaveAttribute("aria-selected", "true"); + expect(screen.queryByLabelText("Database filter")).not.toBeInTheDocument(); + await user.click(await screen.findByRole("button", { name: "Edit description for patients" })); + + expect(screen.getByLabelText("Physical table name")).toHaveValue("patients"); + expect(screen.getByLabelText("Physical table name")).toHaveAttribute("readonly"); + expect(screen.getByLabelText("Source comment")).toHaveAttribute("readonly"); + await user.type( + screen.getByLabelText(/^Description/), + "Registry used for longitudinal patient analysis", + ); + await user.click(screen.getByRole("button", { name: "Save metadata" })); + + await waitFor(() => expect(patchBody).toEqual({ + version: 1, + description: "Registry used for longitudinal patient analysis", + generatedDescription: null, + })); + await user.click(screen.getByRole("button", { name: "Back to tables" })); + await user.click(screen.getByRole("tab", { name: "Overview" })); + expect(await screen.findByRole("heading", { name: "Database details" })).toBeVisible(); +}); + +test("navigates from a table to columns and from the database to physical relationships", async () => { + const user = userEvent.setup(); + let columnPatch: unknown; + const idColumn: CatalogColumn = { + id: "dddddddd-dddd-4ddd-8ddd-dddddddddddd", + tableId: patientsTable.id, + name: "id", + ordinalPosition: 1, + dataType: "bigint", + isNullable: false, + defaultExpression: null, + primaryKeyPosition: 1, + isPrimaryKey: true, + isForeignKey: true, + foreignKeyCount: 1, + sourceComment: "Patient identifier", + description: null, + generatedDescription: null, + lastSyncedDatabaseVersion: 3, + lastSyncedAt: "2026-08-27T10:00:00Z", + version: 1, + createdAt: "2026-08-27T10:00:00Z", + updatedAt: "2026-08-27T10:00:00Z", + }; + server.use( + http.get("/api/catalog/databases/:databaseId/tables", () => HttpResponse.json([patientsTable])), + http.get("/api/catalog/databases/:databaseId/tables/:tableId/columns", () => HttpResponse.json([idColumn])), + http.patch("/api/catalog/databases/:databaseId/tables/:tableId/columns/:columnId", async ({ request }) => { + columnPatch = await request.json(); + return HttpResponse.json({ ...idColumn, generatedDescription: "Generated identifier draft", version: 2 }); + }), + http.get("/api/catalog/databases/:databaseId/relationships", () => HttpResponse.json([{ + id: "eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee", + databaseId: patientsTable.databaseId, + constraintName: "visits_patient_id_fkey", + sourceTableId: "ffffffff-ffff-4fff-8fff-ffffffffffff", + sourceTableName: "visits", + targetTableId: patientsTable.id, + targetTableName: "patients", + updateRule: "NO ACTION", + deleteRule: "CASCADE", + deferrable: false, + initiallyDeferred: false, + columns: [{ position: 1, sourceColumnId: "1", sourceColumnName: "patient_id", targetColumnId: idColumn.id, targetColumnName: "id" }], + lastSyncedDatabaseVersion: 3, + lastSyncedAt: "2026-08-27T10:00:00Z", + createdAt: "2026-08-27T10:00:00Z", + updatedAt: "2026-08-27T10:00:00Z", + }])), + ); + renderPage({ rows: [makeDatabase({ connectionStatus: "reachable", testedVersion: 3 })] }); + + await user.click(await screen.findByRole("button", { name: "View Policlinico San Donato" })); + await user.click(screen.getByRole("tab", { name: "Tables" })); + await user.click(await screen.findByRole("button", { name: "View columns for patients" })); + expect(screen.getByRole("tab", { name: "Columns" })).toHaveAttribute("aria-selected", "true"); + expect(await screen.findByText("PK")).toBeVisible(); + expect(screen.getByText("FK")).toBeVisible(); + await user.click(screen.getByRole("button", { name: "Edit metadata for id" })); + await user.type(screen.getByLabelText("Generated description"), "Generated identifier draft"); + await user.click(screen.getByRole("button", { name: "Save metadata" })); + await waitFor(() => expect(columnPatch).toEqual({ + version: 1, + description: null, + generatedDescription: "Generated identifier draft", + })); + await user.click(screen.getByRole("tab", { name: "Relationships" })); + expect(await screen.findByText("visits_patient_id_fkey")).toBeVisible(); +}); + +test("opens the durable job drawer and confirms its exact destructive plan", async () => { + const user = userEvent.setup(); + const startBodies: unknown[] = []; + const confirmationBodies: unknown[] = []; + const queued: CatalogSyncRun = { + id: "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb", + databaseId: "11111111-1111-4111-8111-111111111111", + scope: "tables", + tableIds: [], + state: "queued", + phase: "queued", + requestedDatabaseVersion: 3, + plannedDiff: null, + confirmationToken: null, + counts: {}, + errorCode: null, + errorMessage: null, + cancelRequested: false, + createdAt: "2026-08-27T10:00:00Z", + startedAt: null, + updatedAt: "2026-08-27T10:00:00Z", + finishedAt: null, + heartbeatAt: null, + }; + const waiting: CatalogSyncRun = { + ...queued, + state: "awaiting_confirmation", + phase: "awaiting_confirmation", + confirmationToken: "cccccccc-cccc-4ccc-8ccc-cccccccccccc", + plannedDiff: { deletedTables: ["legacy_visits"], deletedColumns: [], deletedRelationships: [] }, + counts: { tables: 1 }, + startedAt: "2026-08-27T10:00:00Z", + updatedAt: "2026-08-27T10:00:01Z", + }; + server.use( + http.get("/api/catalog/databases/:databaseId/tables", () => HttpResponse.json([patientsTable])), + http.post("/api/catalog/databases/:databaseId/sync-runs", async ({ request }) => { + startBodies.push(await request.json()); + return HttpResponse.json(queued, { status: 202 }); + }), + http.get("/api/catalog/sync-runs/:runId", () => HttpResponse.json(waiting)), + http.get("/api/catalog/sync-runs/:runId/events-list", () => HttpResponse.json([])), + http.get("/api/catalog/databases/:databaseId/sync-runs", () => HttpResponse.json([waiting])), + http.post("/api/catalog/sync-runs/:runId/confirm", async ({ request }) => { + confirmationBodies.push(await request.json()); + return HttpResponse.json({ ...waiting, state: "queued", phase: "queued", confirmationToken: null }); + }), + ); + renderPage({ + rows: [makeDatabase({ connectionStatus: "reachable", testedVersion: 3 })], + }); + + await user.click(await screen.findByRole("button", { name: "View Policlinico San Donato" })); + await user.click(screen.getByRole("tab", { name: "Tables" })); + await user.click(await screen.findByRole("button", { name: "Sync tables" })); + + const synchronizationDrawer = await screen.findByRole("complementary", { + name: "Schema synchronization", + }); + expect(synchronizationDrawer).toHaveClass("inset-y-2", "sm:inset-y-4"); + expect(within(synchronizationDrawer).getByRole("listitem", { + name: "Confirmation required, in progress", + })).toHaveAttribute("aria-current", "step"); + expect(await screen.findByText("table · legacy_visits")).toBeVisible(); + expect(startBodies).toEqual([{ version: 3, scope: "tables", tableIds: [] }]); + await user.click(screen.getByRole("button", { name: "Confirm removals" })); + await waitFor(() => expect(confirmationBodies).toEqual([ + { confirmationToken: "cccccccc-cccc-4ccc-8ccc-cccccccccccc" }, + ])); +}); + +test("keeps a stale table draft but requires an explicit reload before another save", async () => { + const user = userEvent.setup(); + const current = { + ...patientsTable, + description: "Description saved by another editor", + version: 2, + }; + let latest = patientsTable; + server.use( + http.get("/api/catalog/databases/:databaseId/tables", () => HttpResponse.json([latest])), + http.patch("/api/catalog/databases/:databaseId/tables/:tableId", () => { + latest = current; + return HttpResponse.json({ + code: "table_stale", + message: "Table description changed. Reload and try again.", + }, { status: 409 }); + }), + ); + renderPage({ + rows: [makeDatabase({ connectionStatus: "reachable", testedVersion: 3 })], + }); + + await user.click(await screen.findByRole("button", { name: "View Policlinico San Donato" })); + await user.click(screen.getByRole("tab", { name: "Tables" })); + await user.click(await screen.findByRole("button", { name: "Edit description for patients" })); + const description = screen.getByLabelText(/^Description/); + await user.type(description, "My unsaved draft"); + await user.click(screen.getByRole("button", { name: "Save metadata" })); + + expect(await screen.findByText("A newer table description is available.")).toBeVisible(); + expect(description).toHaveValue("My unsaved draft"); + expect(screen.getByRole("button", { name: "Save metadata" })).toBeDisabled(); + await user.click(screen.getByRole("button", { name: "Keep editing" })); + expect(screen.getByText("Reload the latest value before this description can be saved.")).toBeVisible(); + await user.click(screen.getByRole("button", { name: "Reload latest" })); + + await waitFor(() => expect(description).toHaveValue("Description saved by another editor")); + expect(screen.getByRole("button", { name: "Save metadata" })).toBeDisabled(); +}); diff --git a/frontend/src/shell/DatabaseManagementPage.tsx b/frontend/src/shell/DatabaseManagementPage.tsx index f30306d7..857a1569 100644 --- a/frontend/src/shell/DatabaseManagementPage.tsx +++ b/frontend/src/shell/DatabaseManagementPage.tsx @@ -1,8 +1,756 @@ -export function DatabaseManagementPage() { +import { + useCallback, + useEffect, + useMemo, + useRef, + useState, +} from "react"; +import { useQuery, useQueryClient } from "@tanstack/react-query"; +import { Plus, RefreshCw } from "lucide-react"; +import { toast } from "sonner"; +import { Button } from "../components/ui/button"; +import { ApiError, apiErrorMessage } from "../api/client"; +import { + createCatalogDatabase, + deleteCatalogDatabase, + listCatalogDatabases, + replaceCatalogDatabaseSecrets, + startCatalogSync, + testCatalogDatabase, + updateCatalogDatabase, + type CatalogDatabase, + type CatalogSecretName, + type CatalogSyncScope, + type CatalogSyncRun, + type DatabaseBinding, + type DatabaseTransport, +} from "../api/catalog-databases"; +import { DatabaseGrid } from "./database-management/DatabaseGrid"; +import { DatabaseForm } from "./database-management/DatabaseForm"; +import { DatabaseTables } from "./database-management/DatabaseTables"; +import { DatabaseRelationships } from "./database-management/DatabaseRelationships"; +import { CatalogSyncDrawer } from "./database-management/CatalogSyncDrawer"; +import { + configurationFingerprint, + configurationFromDraft, + draftFrom, + hasSecretChanges, + secretReplacements, + type DatabaseBusyAction, + type DatabaseFormDraft, + type DatabaseFormMode, + type DatabaseNavigationState, + type DatabaseScreen, +} from "./database-management/model"; + +const DATABASE_QUERY_KEY = ["catalog-databases"] as const; +const SYNC_STARTED_MESSAGES: Record = { + tables: "Table synchronization started", + columns: "Column synchronization started", + relationships: "Relationship synchronization started", + all: "Full schema synchronization started", +}; + +interface Props { + canManage: boolean; + canManageSecrets: boolean; + onNavigationStateChange?: (state: DatabaseNavigationState) => void; +} + +interface FormSource { + id?: string; + version: number; +} + +function isStaleError(error: unknown): boolean { + return error instanceof ApiError && error.code === "database_stale"; +} + +export function DatabaseManagementPage({ + canManage, + canManageSecrets, + onNavigationStateChange, +}: Props) { + const queryClient = useQueryClient(); + const { + data, + isLoading, + isError, + isFetching, + refetch, + } = useQuery({ + queryKey: DATABASE_QUERY_KEY, + queryFn: listCatalogDatabases, + retry: false, + }); + const rows = data ?? []; + + const [screen, setScreen] = useState({ kind: "list" }); + const [draft, setDraft] = useState(null); + const [baseline, setBaseline] = useState(""); + const [formSource, setFormSource] = useState(null); + const [search, setSearch] = useState(""); + const [busyAction, setBusyAction] = useState(null); + const [stale, setStale] = useState(false); + const [staleBannerOpen, setStaleBannerOpen] = useState(true); + const [partialSecretFailure, setPartialSecretFailure] = useState(null); + const [tablesNavigationState, setTablesNavigationState] = useState({ + dirty: false, + busy: false, + }); + const [activeSyncRun, setActiveSyncRun] = useState(null); + const [syncDrawerOpen, setSyncDrawerOpen] = useState(false); + + const originRef = useRef(null); + const searchInputRef = useRef(null); + const formHeadingRef = useRef(null); + + const activeRow = screen.kind === "list" + ? undefined + : rows.find((row) => row.workspaceId === screen.workspaceId); + const availableWorkspaces = useMemo( + () => rows.filter((row) => row.workspaceAvailable && !row.configured), + [rows], + ); + const editable = screen.kind === "add" || screen.kind === "edit"; + const configurationDirty = Boolean( + editable + && draft + && configurationFingerprint(draft) !== baseline, + ); + const dirty = Boolean(editable && draft && (configurationDirty || hasSecretChanges(draft))); + const busy = busyAction !== null; + const navigationDirty = screen.kind === "tables" ? tablesNavigationState.dirty : dirty; + const navigationBusy = screen.kind === "tables" ? tablesNavigationState.busy : busy; + + useEffect(() => { + onNavigationStateChange?.({ dirty: navigationDirty, busy: navigationBusy }); + }, [navigationBusy, navigationDirty, onNavigationStateChange]); + + useEffect(() => () => { + onNavigationStateChange?.({ dirty: false, busy: false }); + }, [onNavigationStateChange]); + + useEffect(() => { + const warn = (event: BeforeUnloadEvent) => { + if (!navigationDirty && !navigationBusy) return; + event.preventDefault(); + }; + window.addEventListener("beforeunload", warn); + return () => window.removeEventListener("beforeunload", warn); + }, [navigationBusy, navigationDirty]); + + useEffect(() => { + if (screen.kind === "list" || !activeRow || !formSource || busy) return; + if (activeRow.id === formSource.id && activeRow.version === formSource.version) return; + setStale(true); + setStaleBannerOpen(true); + }, [activeRow, busy, formSource, screen.kind]); + + useEffect(() => { + if (screen.kind === "list") return; + const timer = window.setTimeout(() => formHeadingRef.current?.focus(), 0); + return () => window.clearTimeout(timer); + }, [screen.kind, screen.kind === "list" ? "" : screen.workspaceId]); + + const cacheSavedRow = useCallback((saved: CatalogDatabase) => { + queryClient.setQueryData(DATABASE_QUERY_KEY, (current = []) => { + const existing = current.findIndex((row) => row.workspaceId === saved.workspaceId); + if (existing < 0) return [...current, saved]; + return current.map((row, index) => index === existing ? saved : row); + }); + }, [queryClient]); + + const restoreListFocus = useCallback(() => { + window.setTimeout(() => { + if (originRef.current?.isConnected) { + originRef.current.focus(); + } else { + searchInputRef.current?.focus(); + } + }, 0); + }, []); + + const showList = useCallback(() => { + setScreen({ kind: "list" }); + setDraft(null); + setBaseline(""); + setFormSource(null); + setStale(false); + setStaleBannerOpen(true); + setPartialSecretFailure(null); + setTablesNavigationState({ dirty: false, busy: false }); + restoreListFocus(); + }, [restoreListFocus]); + + useEffect(() => { + if (screen.kind === "list" || isLoading || activeRow) return; + showList(); + toast.info("This database configuration is no longer available"); + }, [activeRow, isLoading, screen.kind, showList]); + + useEffect(() => { + if (!["tables", "relationships"].includes(screen.kind) || !activeRow || (activeRow.configured && activeRow.id)) return; + showList(); + toast.info("Save the database configuration before managing tables"); + }, [activeRow, screen.kind, showList]); + + const backToList = useCallback(() => { + if (busy) return; + if (dirty && !window.confirm("Discard unsaved database changes?")) return; + showList(); + }, [busy, dirty, showList]); + + const openForm = useCallback(( + mode: DatabaseFormMode, + row: CatalogDatabase, + origin: HTMLElement, + workspaceLocked = false, + ) => { + const nextDraft = draftFrom(row); + originRef.current = origin; + setDraft(nextDraft); + setBaseline(configurationFingerprint(nextDraft)); + setFormSource({ id: row.id, version: row.version }); + setStale(false); + setStaleBannerOpen(true); + setPartialSecretFailure(null); + setScreen({ + kind: mode, + workspaceId: row.workspaceId, + ...(mode === "add" ? { workspaceLocked } : {}), + }); + }, []); + + const viewRow = useCallback((row: CatalogDatabase, origin: HTMLButtonElement) => { + openForm("view", row, origin); + }, [openForm]); + + const editRow = useCallback((row: CatalogDatabase, origin: HTMLButtonElement) => { + openForm(row.configured ? "edit" : "add", row, origin, !row.configured); + }, [openForm]); + + const deleteRow = useCallback((row: CatalogDatabase, origin: HTMLButtonElement) => { + openForm("delete", row, origin); + }, [openForm]); + + const openTables = useCallback((row: CatalogDatabase, origin?: HTMLElement) => { + if (!row.configured || !row.id) return; + if (origin) originRef.current = origin; + setDraft(null); + setBaseline(""); + setFormSource(null); + setStale(false); + setPartialSecretFailure(null); + setTablesNavigationState({ dirty: false, busy: false }); + setScreen({ kind: "tables", workspaceId: row.workspaceId }); + }, []); + + const openRelationships = useCallback((row: CatalogDatabase) => { + if (!row.configured || !row.id) return; + setDraft(null); + setBaseline(""); + setFormSource(null); + setStale(false); + setPartialSecretFailure(null); + setTablesNavigationState({ dirty: false, busy: false }); + setScreen({ kind: "relationships", workspaceId: row.workspaceId }); + }, []); + + const openOverview = useCallback((row: CatalogDatabase) => { + const nextDraft = draftFrom(row); + setDraft(nextDraft); + setBaseline(configurationFingerprint(nextDraft)); + setFormSource({ id: row.id, version: row.version }); + setStale(false); + setStaleBannerOpen(true); + setPartialSecretFailure(null); + setTablesNavigationState({ dirty: false, busy: false }); + setScreen({ kind: "view", workspaceId: row.workspaceId }); + }, []); + + const addDatabase = useCallback((origin: HTMLElement) => { + const workspace = availableWorkspaces[0]; + if (!workspace || !canManage) return; + openForm("add", workspace, origin, false); + }, [availableWorkspaces, canManage, openForm]); + + const changeWorkspace = useCallback((workspaceId: string) => { + const workspace = availableWorkspaces.find((row) => row.workspaceId === workspaceId); + if (!workspace) return; + if (dirty && !window.confirm("Discard changes and choose another workspace?")) return; + const nextDraft = draftFrom(workspace); + setDraft(nextDraft); + setBaseline(configurationFingerprint(nextDraft)); + setFormSource({ id: workspace.id, version: workspace.version }); + setStale(false); + setPartialSecretFailure(null); + setScreen({ kind: "add", workspaceId, workspaceLocked: false }); + }, [availableWorkspaces, dirty]); + + const changeField = useCallback((field: "databaseName" | "schema", value: string) => { + setDraft((current) => current ? { ...current, [field]: value } : current); + }, []); + + const changeTransport = useCallback((transport: DatabaseTransport) => { + setDraft((current) => current + ? { ...current, binding: { ...current.binding, transport } } + : current); + }, []); + + const changeBinding = useCallback(( + key: K, + value: DatabaseBinding[K], + ) => { + setDraft((current) => current + ? { ...current, binding: { ...current.binding, [key]: value } } + : current); + }, []); + + const changeSecret = useCallback((name: CatalogSecretName, value: string) => { + setDraft((current) => current + ? { ...current, secrets: { ...current.secrets, [name]: value } } + : current); + }, []); + + const markStale = useCallback(() => { + setStale(true); + setStaleBannerOpen(true); + }, []); + + const save = useCallback(async () => { + if (!activeRow || !formSource || !draft || !editable || !canManage || stale || busy) return; + setBusyAction("save"); + setPartialSecretFailure(null); + + try { + const input = configurationFromDraft(draft); + let saved = activeRow; + const shouldPersistConfiguration = !formSource.id || configurationDirty; + + if (shouldPersistConfiguration) { + saved = formSource.id + ? await updateCatalogDatabase(formSource.id, formSource.version, input) + : await createCatalogDatabase(input); + setFormSource({ id: saved.id, version: saved.version }); + cacheSavedRow(saved); + } + + const replacements = secretReplacements(draft); + if (Object.keys(replacements).length > 0) { + if (!canManageSecrets) throw new Error("Secret replacement is not permitted"); + try { + saved = await replaceCatalogDatabaseSecrets( + saved.id ?? formSource.id!, + shouldPersistConfiguration ? saved.version : formSource.version, + replacements, + ); + setFormSource({ id: saved.id, version: saved.version }); + cacheSavedRow(saved); + } catch (error) { + setBaseline(configurationFingerprint(draft)); + setScreen({ kind: "edit", workspaceId: saved.workspaceId }); + setPartialSecretFailure(apiErrorMessage(error)); + if (isStaleError(error)) { + markStale(); + } else { + await queryClient.invalidateQueries({ queryKey: DATABASE_QUERY_KEY }); + } + toast.warning("Database configuration saved, but secrets still need attention"); + return; + } + } + + const nextDraft = draftFrom(saved); + setDraft(nextDraft); + setBaseline(configurationFingerprint(nextDraft)); + setFormSource({ id: saved.id, version: saved.version }); + setScreen({ kind: "edit", workspaceId: saved.workspaceId }); + setStale(false); + setPartialSecretFailure(null); + await queryClient.invalidateQueries({ queryKey: DATABASE_QUERY_KEY }); + toast.success(activeRow.configured ? "Database configuration saved" : "Database configuration added"); + } catch (error) { + if (isStaleError(error)) { + markStale(); + } else { + toast.error(apiErrorMessage(error)); + } + } finally { + setBusyAction(null); + } + }, [ + activeRow, + busy, + cacheSavedRow, + canManage, + canManageSecrets, + configurationDirty, + draft, + editable, + formSource, + markStale, + queryClient, + stale, + ]); + + const retrySecrets = useCallback(async () => { + if (!activeRow?.configured || !formSource?.id || !draft || !canManageSecrets || busy || stale) return; + const replacements = secretReplacements(draft); + if (Object.keys(replacements).length === 0) { + setPartialSecretFailure(null); + return; + } + + setBusyAction("retry-secrets"); + try { + const saved = await replaceCatalogDatabaseSecrets(formSource.id, formSource.version, replacements); + setFormSource({ id: saved.id, version: saved.version }); + cacheSavedRow(saved); + setDraft({ ...draft, secrets: {} }); + setPartialSecretFailure(null); + await queryClient.invalidateQueries({ queryKey: DATABASE_QUERY_KEY }); + toast.success("Database secrets saved"); + } catch (error) { + setPartialSecretFailure(apiErrorMessage(error)); + if (isStaleError(error)) markStale(); + toast.error(apiErrorMessage(error)); + } finally { + setBusyAction(null); + } + }, [activeRow, busy, cacheSavedRow, canManageSecrets, draft, formSource, markStale, queryClient, stale]); + + const testConnection = useCallback(async () => { + if ( + !activeRow?.configured + || !formSource?.id + || !draft + || !canManage + || dirty + || stale + || busy + ) return; + + setBusyAction("test"); + try { + const tested = await testCatalogDatabase(formSource.id, formSource.version); + setFormSource({ id: tested.id, version: tested.version }); + cacheSavedRow(tested); + const nextDraft = draftFrom(tested); + setDraft(nextDraft); + setBaseline(configurationFingerprint(nextDraft)); + await queryClient.invalidateQueries({ queryKey: DATABASE_QUERY_KEY }); + if (tested.connectionStatus === "reachable") { + toast.success("Database connection is reachable"); + } else { + toast.error(tested.lastErrorMessage ?? "Database connection failed"); + } + } catch (error) { + if (isStaleError(error)) markStale(); + else toast.error(apiErrorMessage(error)); + } finally { + setBusyAction(null); + } + }, [activeRow, busy, cacheSavedRow, canManage, dirty, draft, formSource, markStale, queryClient, stale]); + + const remove = useCallback(async () => { + if (!activeRow?.configured || !formSource?.id || !canManage || busy || stale) return; + setBusyAction("delete"); + try { + await deleteCatalogDatabase(formSource.id, formSource.version); + queryClient.setQueryData(DATABASE_QUERY_KEY, (current = []) => { + if (!activeRow.workspaceAvailable) { + return current.filter((row) => row.workspaceId !== activeRow.workspaceId); + } + return current.map((row) => row.workspaceId === activeRow.workspaceId + ? { + ...row, + id: undefined, + configured: false, + version: 0, + createdAt: "", + updatedAt: "", + connectionStatus: "untested", + testedVersion: undefined, + lastTestedAt: undefined, + lastErrorCode: undefined, + lastErrorMessage: undefined, + secrets: { + password: false, + apiKey: false, + sshPrivateKey: false, + sshPrivateKeyPassphrase: false, + sshKnownHosts: false, + tlsCa: false, + }, + } + : row); + }); + setBusyAction(null); + showList(); + await queryClient.invalidateQueries({ queryKey: DATABASE_QUERY_KEY }); + toast.success("Database configuration deleted"); + } catch (error) { + if (isStaleError(error)) markStale(); + else toast.error(apiErrorMessage(error)); + } finally { + setBusyAction(null); + } + }, [activeRow, busy, canManage, formSource, markStale, queryClient, showList, stale]); + + const reloadLatest = useCallback(async () => { + if (busy || screen.kind === "list") return; + setBusyAction("reload"); + try { + const result = await refetch(); + if (result.error) throw result.error; + const latest = result.data?.find((row) => row.workspaceId === screen.workspaceId); + if (!latest) { + showList(); + toast.info("This database configuration is no longer available"); + return; + } + const nextDraft = draftFrom(latest); + setDraft(nextDraft); + setBaseline(configurationFingerprint(nextDraft)); + setFormSource({ id: latest.id, version: latest.version }); + setStale(false); + setStaleBannerOpen(true); + setPartialSecretFailure(null); + if (!latest.configured && screen.kind === "delete") { + showList(); + toast.info("This database configuration has already been deleted"); + } else if (screen.kind === "edit" && !latest.configured) { + setScreen({ kind: "add", workspaceId: latest.workspaceId, workspaceLocked: true }); + } + } catch (error) { + toast.error(apiErrorMessage(error)); + } finally { + setBusyAction(null); + } + }, [busy, refetch, screen, showList]); + + const refreshList = useCallback(async () => { + if (isFetching) return; + try { + const result = await refetch(); + if (result.error) throw result.error; + } catch (error) { + toast.error(apiErrorMessage(error)); + } + }, [isFetching, refetch]); + + const updateTrackedSyncRun = useCallback((run: CatalogSyncRun) => { + setActiveSyncRun(run); + queryClient.setQueryData(DATABASE_QUERY_KEY, (current = []) => current.map((row) => ( + row.id === run.databaseId + ? { ...row, activeSyncRun: ["queued", "running", "awaiting_confirmation", "applying"].includes(run.state) ? run : undefined } + : row + ))); + }, [queryClient]); + + const rememberSyncRun = useCallback((run: CatalogSyncRun) => { + updateTrackedSyncRun(run); + setSyncDrawerOpen(true); + }, [updateTrackedSyncRun]); + + const openSync = useCallback((row?: CatalogDatabase) => { + const run = row?.activeSyncRun ?? activeSyncRun; + if (!run) return; + setActiveSyncRun(run); + setSyncDrawerOpen(true); + }, [activeSyncRun]); + + const testSelected = useCallback(async (selected: CatalogDatabase[]) => { + try { + const tested = await Promise.all(selected.map((row) => testCatalogDatabase(row.id!, row.version))); + for (const row of tested) cacheSavedRow(row); + await queryClient.invalidateQueries({ queryKey: DATABASE_QUERY_KEY }); + toast.success(`${tested.length} connection${tested.length === 1 ? "" : "s"} tested`); + } catch (error) { + toast.error(apiErrorMessage(error)); + throw error; + } + }, [cacheSavedRow, queryClient]); + + const syncSelected = useCallback(async (selected: CatalogDatabase[], scope: CatalogSyncScope) => { + try { + const runs = await Promise.all(selected.map((row) => startCatalogSync(row.id!, row.version, scope))); + queryClient.setQueryData(DATABASE_QUERY_KEY, (current = []) => current.map((row) => { + const run = runs.find((candidate) => candidate.databaseId === row.id); + return run ? { ...row, activeSyncRun: run } : row; + })); + if (runs[0]) rememberSyncRun(runs[0]); + toast.success(`${runs.length} schema synchronization${runs.length === 1 ? "" : "s"} started`); + } catch (error) { + toast.error(apiErrorMessage(error)); + throw error; + } + }, [queryClient, rememberSyncRun]); + + const syncDatabase = useCallback(async (scope: CatalogSyncScope) => { + if (!activeRow?.id || !activeRow.configured) return; + try { + rememberSyncRun(await startCatalogSync(activeRow.id, activeRow.version, scope)); + toast.success(SYNC_STARTED_MESSAGES[scope]); + } catch (error) { toast.error(apiErrorMessage(error)); } + }, [activeRow, rememberSyncRun]); + + const catalogChanged = useCallback(async () => { + const databaseId = activeSyncRun?.databaseId; + if (databaseId) { + await Promise.all([ + queryClient.invalidateQueries({ queryKey: ["catalog-tables", databaseId] }), + queryClient.invalidateQueries({ queryKey: ["catalog-relationships", databaseId] }), + queryClient.invalidateQueries({ queryKey: ["catalog-sync-runs", databaseId] }), + ]); + } + await queryClient.invalidateQueries({ queryKey: DATABASE_QUERY_KEY }); + }, [activeSyncRun, queryClient]); + + const formVisible = screen.kind !== "list" && screen.kind !== "tables" && screen.kind !== "relationships" && activeRow && draft; + const tablesVisible = screen.kind === "tables" && activeRow?.configured && activeRow.id; + const relationshipsVisible = screen.kind === "relationships" && activeRow?.configured && activeRow.id; + const currentActiveRun = activeSyncRun && ["queued", "running", "awaiting_confirmation", "applying"].includes(activeSyncRun.state) + ? activeSyncRun + : activeRow?.activeSyncRun && ["queued", "running", "awaiting_confirmation", "applying"].includes(activeRow.activeSyncRun.state) + ? activeRow.activeSyncRun + : undefined; + return ( -
+
+
+
+

Administration

+

Database management

+

+ One database configuration for each repository workspace. +

+
+ {screen.kind === "list" ? ( +
+ + +
+ ) : null} +
+ +
+
+ {isError && rows.length === 0 ? ( +
+
+

The database catalog is unavailable.

+

Verify the catalog service and database migrations, then try again.

+ +
+
+ ) : ( + + )} +
+ + {formVisible ? ( + void save()} + onTest={() => void testConnection()} + onDelete={() => void remove()} + onReloadLatest={() => void reloadLatest()} + onKeepEditing={() => setStaleBannerOpen(false)} + onRetrySecrets={() => void retrySecrets()} + onOpenTables={(origin) => openTables(activeRow, origin)} + onOpenRelationships={() => openRelationships(activeRow)} + onSync={(scope) => void syncDatabase(scope)} + onOpenSync={() => openSync(activeRow)} + activeSyncRun={currentActiveRun} + /> + ) : null} + + {tablesVisible ? ( + openOverview(activeRow)} + onOpenRelationships={() => openRelationships(activeRow)} + onNavigationStateChange={setTablesNavigationState} + onRunStarted={rememberSyncRun} + onOpenSync={() => openSync(activeRow)} + /> + ) : null} + + {relationshipsVisible ? ( + openOverview(activeRow)} + onOpenTables={() => openTables(activeRow)} + onRunStarted={rememberSyncRun} + onOpenSync={() => openSync(activeRow)} + /> + ) : null} +
+ setSyncDrawerOpen(false)} + onRunChange={rememberSyncRun} + onRunUpdate={updateTrackedSyncRun} + onCatalogChanged={() => void catalogChanged()} + /> +
); } diff --git a/frontend/src/shell/database-management/CatalogSyncDrawer.tsx b/frontend/src/shell/database-management/CatalogSyncDrawer.tsx new file mode 100644 index 00000000..822d5b06 --- /dev/null +++ b/frontend/src/shell/database-management/CatalogSyncDrawer.tsx @@ -0,0 +1,402 @@ +import { useEffect, useMemo, useRef, useState } from "react"; +import { useQuery, useQueryClient } from "@tanstack/react-query"; +import { AlertTriangle, Check, CheckCircle2, Circle, LoaderCircle, Minus, RotateCcw, X } from "lucide-react"; +import { toast } from "sonner"; +import { Button } from "../../components/ui/button"; +import { apiErrorMessage } from "../../api/client"; +import { + cancelCatalogSync, + catalogSyncEventsUrl, + confirmCatalogSync, + getCatalogSyncRun, + listCatalogSyncEvents, + listCatalogSyncRuns, + retryCatalogSync, + type CatalogSyncEvent, + type CatalogSyncPhase, + type CatalogSyncRun, +} from "../../api/catalog-databases"; + +interface Props { + databaseId: string | null; + runId: string | null; + open: boolean; + onClose: () => void; + onRunChange: (run: CatalogSyncRun) => void; + onRunUpdate: (run: CatalogSyncRun) => void; + onCatalogChanged: () => void; +} + +const phases: CatalogSyncPhase[] = [ + "queued", "connecting", "scanning_tables", "scanning_columns", + "scanning_relationships", "planning", "awaiting_confirmation", "applying", "completed", +]; + +const phaseEnteredByEvent: Record = { + queued: "queued", + started: "connecting", + connecting: "connecting", + scanning_tables: "scanning_tables", + scanning_columns: "scanning_columns", + scanning_relationships: "scanning_relationships", + planning: "planning", + confirmation_required: "awaiting_confirmation", + confirmation_received: "awaiting_confirmation", + applying: "applying", + succeeded: "completed", +}; + +const phaseLabels: Record = { + queued: { pending: "Queue", current: "Queued", completed: "Queued" }, + connecting: { pending: "Connect", current: "Connecting", completed: "Connected" }, + scanning_tables: { pending: "Read tables", current: "Reading tables", completed: "Tables read" }, + scanning_columns: { pending: "Read columns", current: "Reading columns", completed: "Columns read" }, + scanning_relationships: { + pending: "Read relationships", + current: "Reading relationships", + completed: "Relationships read", + }, + planning: { pending: "Plan changes", current: "Planning changes", completed: "Changes planned" }, + awaiting_confirmation: { + pending: "Confirm changes if needed", + current: "Confirmation required", + completed: "Changes confirmed", + }, + applying: { pending: "Apply changes", current: "Applying changes", completed: "Changes applied" }, + completed: { pending: "Complete", current: "Completing", completed: "Completed" }, +}; + +function terminal(run?: CatalogSyncRun): boolean { + return Boolean(run && ["succeeded", "failed", "cancelled", "interrupted"].includes(run.state)); +} + +function elapsed(run: CatalogSyncRun, now: number): string { + const start = new Date(run.startedAt ?? run.createdAt).getTime(); + const end = run.finishedAt ? new Date(run.finishedAt).getTime() : now; + const seconds = Math.max(0, Math.floor((end - start) / 1000)); + const minutes = Math.floor(seconds / 60); + return minutes ? `${minutes}m ${seconds % 60}s` : `${seconds}s`; +} + +function stateLabel(run: CatalogSyncRun): string { + return run.state.replaceAll("_", " "); +} + +export function CatalogSyncDrawer({ + databaseId, + runId, + open, + onClose, + onRunChange, + onRunUpdate, + onCatalogChanged, +}: Props) { + const queryClient = useQueryClient(); + const [events, setEvents] = useState([]); + const [now, setNow] = useState(Date.now()); + const [action, setAction] = useState<"confirm" | "cancel" | "retry" | null>(null); + const lastSequence = useRef(0); + const notifiedRun = useRef(null); + + const runQuery = useQuery({ + queryKey: ["catalog-sync-run", runId], + queryFn: () => getCatalogSyncRun(runId!), + enabled: Boolean(runId), + retry: false, + refetchInterval: (query) => terminal(query.state.data) ? false : 1_000, + }); + const run = runQuery.data; + const historyQuery = useQuery({ + queryKey: ["catalog-sync-runs", databaseId], + queryFn: () => listCatalogSyncRuns(databaseId!), + enabled: Boolean(databaseId && open), + retry: false, + refetchInterval: run && !terminal(run) ? 2_000 : false, + }); + const eventQuery = useQuery({ + queryKey: ["catalog-sync-events", runId], + queryFn: () => listCatalogSyncEvents(runId!, lastSequence.current), + enabled: Boolean(runId && open), + retry: false, + refetchInterval: run && terminal(run) ? false : 1_500, + }); + + const mergeEvents = (incoming: CatalogSyncEvent[]) => { + if (incoming.length === 0) return; + setEvents((current) => { + const bySequence = new Map(current.map((event) => [event.sequence, event])); + for (const event of incoming) bySequence.set(event.sequence, event); + const merged = [...bySequence.values()].sort((a, b) => a.sequence - b.sequence); + lastSequence.current = merged.at(-1)?.sequence ?? lastSequence.current; + return merged; + }); + }; + + useEffect(() => { + setEvents([]); + lastSequence.current = 0; + }, [runId]); + + useEffect(() => { mergeEvents(eventQuery.data ?? []); }, [eventQuery.data]); + + useEffect(() => { + if (!runId || !open || typeof EventSource === "undefined") return; + const source = new EventSource(catalogSyncEventsUrl(runId)); + const log = (event: MessageEvent) => { + try { mergeEvents([JSON.parse(event.data) as CatalogSyncEvent]); } catch { /* polling remains authoritative */ } + }; + const update = (event: MessageEvent) => { + try { + const next = JSON.parse(event.data) as CatalogSyncRun; + queryClient.setQueryData(["catalog-sync-run", runId], next); + } catch { /* polling remains authoritative */ } + }; + source.addEventListener("log", log as EventListener); + source.addEventListener("run", update as EventListener); + source.onerror = () => source.close(); + return () => source.close(); + }, [open, queryClient, runId]); + + useEffect(() => { + if (!run || terminal(run)) return; + const timer = window.setInterval(() => setNow(Date.now()), 1_000); + return () => window.clearInterval(timer); + }, [run]); + + useEffect(() => { + if (!run) return; + onRunUpdate(run); + queryClient.setQueryData(["catalog-sync-runs", run.databaseId], (current) => { + if (!current) return [run]; + const found = current.some((item) => item.id === run.id); + return found + ? current.map((item) => item.id === run.id ? run : item) + : [run, ...current]; + }); + }, [onRunUpdate, queryClient, run]); + + useEffect(() => { + if (!run || run.state !== "succeeded" || notifiedRun.current === run.id) return; + notifiedRun.current = run.id; + onCatalogChanged(); + }, [onCatalogChanged, run]); + + const currentPhase = run ? phases.indexOf(run.phase) : -1; + const knownEvents = [...events, ...(eventQuery.data ?? [])]; + const confirmationWasRequired = run?.state === "awaiting_confirmation" || knownEvents.some((event) => ( + event.eventType === "confirmation_required" || event.eventType === "confirmation_received" + )); + const confirmationWasReceived = knownEvents.some((event) => event.eventType === "confirmation_received") + || (run?.state === "succeeded" && confirmationWasRequired); + const furthestEventPhase = knownEvents.reduce((furthest, event) => { + const phase = phaseEnteredByEvent[event.eventType]; + return phase ? Math.max(furthest, phases.indexOf(phase)) : furthest; + }, -1); + const confirmationPhase = phases.indexOf("awaiting_confirmation"); + const passedConfirmation = run?.state === "succeeded" + || (run && !terminal(run) ? currentPhase > confirmationPhase : furthestEventPhase > confirmationPhase); + const deletionCount = useMemo(() => run?.plannedDiff + ? run.plannedDiff.deletedTables.length + run.plannedDiff.deletedColumns.length + run.plannedDiff.deletedRelationships.length + : 0, [run]); + + const perform = async (kind: "confirm" | "cancel" | "retry") => { + if (!run) return; + setAction(kind); + try { + const next = kind === "confirm" + ? await confirmCatalogSync(run.id, run.confirmationToken!) + : kind === "cancel" + ? await cancelCatalogSync(run.id) + : await retryCatalogSync(run.id); + onRunChange(next); + queryClient.setQueryData(["catalog-sync-run", next.id], next); + await historyQuery.refetch(); + } catch (error) { + toast.error(apiErrorMessage(error)); + } finally { + setAction(null); + } + }; + + if (!open) return null; + return ( + + ); +} diff --git a/frontend/src/shell/database-management/DatabaseColumns.tsx b/frontend/src/shell/database-management/DatabaseColumns.tsx new file mode 100644 index 00000000..734bb514 --- /dev/null +++ b/frontend/src/shell/database-management/DatabaseColumns.tsx @@ -0,0 +1,192 @@ +import { useEffect, useMemo, useRef, useState } from "react"; +import { useQuery, useQueryClient } from "@tanstack/react-query"; +import { AgGridReact } from "ag-grid-react"; +import type { ColDef, ICellRendererParams } from "ag-grid-community"; +import { KeyRound, Link2, Pencil, RefreshCw, Save } from "lucide-react"; +import { toast } from "sonner"; +import { Button } from "../../components/ui/button"; +import { ApiError, apiErrorMessage } from "../../api/client"; +import { + listCatalogColumns, + updateCatalogColumnMetadata, + type CatalogColumn, + type CatalogTable, +} from "../../api/catalog-databases"; +import type { DatabaseNavigationState } from "./model"; + +interface Props { + databaseId: string; + table: CatalogTable; + canManage: boolean; + onNavigationStateChange: (state: DatabaseNavigationState) => void; + onSync: () => void; +} + +interface GridContext { + canManage: boolean; + onEdit: (column: CatalogColumn, origin: HTMLButtonElement) => void; +} + +function KeyCell({ data }: ICellRendererParams) { + if (!data) return null; + return ( +
+ {data.isPrimaryKey ? PK{data.primaryKeyPosition && data.primaryKeyPosition > 1 ? ` ${data.primaryKeyPosition}` : ""} : null} + {data.isForeignKey ? FK{data.foreignKeyCount > 1 ? ` ${data.foreignKeyCount}` : ""} : null} +
+ ); +} + +function ActionCell({ data, context }: ICellRendererParams) { + if (!data || !context) return null; + return ( +
event.stopPropagation()}> + +
+ ); +} + +export function DatabaseColumns({ databaseId, table, canManage, onNavigationStateChange, onSync }: Props) { + const queryClient = useQueryClient(); + const queryKey = ["catalog-columns", databaseId, table.id] as const; + const { data = [], isLoading, isFetching, refetch } = useQuery({ + queryKey, + queryFn: () => listCatalogColumns(databaseId, table.id), + retry: false, + }); + const [search, setSearch] = useState(""); + const [editingId, setEditingId] = useState(null); + const [description, setDescription] = useState(""); + const [generatedDescription, setGeneratedDescription] = useState(""); + const [baseline, setBaseline] = useState(""); + const [version, setVersion] = useState(null); + const [stale, setStale] = useState(false); + const [staleBannerOpen, setStaleBannerOpen] = useState(true); + const [busy, setBusy] = useState(false); + const originRef = useRef(null); + const active = editingId ? data.find((column) => column.id === editingId) : undefined; + const fingerprint = JSON.stringify([description, generatedDescription]); + const dirty = Boolean(editingId && fingerprint !== baseline); + + useEffect(() => { onNavigationStateChange({ dirty, busy }); }, [busy, dirty, onNavigationStateChange]); + useEffect(() => { + if (!editingId || !active || version === active.version || busy) return; + setStale(true); + setStaleBannerOpen(true); + }, [active, busy, editingId, version]); + + const edit = (column: CatalogColumn, origin: HTMLButtonElement) => { + originRef.current = origin; + setEditingId(column.id); + setDescription(column.description ?? ""); + setGeneratedDescription(column.generatedDescription ?? ""); + setBaseline(JSON.stringify([column.description ?? "", column.generatedDescription ?? ""])); + setVersion(column.version); + setStale(false); + setStaleBannerOpen(true); + }; + const closeEditor = () => { + if (busy) return; + if (dirty && !window.confirm("Discard unsaved column metadata?")) return; + setEditingId(null); + window.setTimeout(() => originRef.current?.focus(), 0); + }; + const save = async () => { + if (!active || version === null) return; + setBusy(true); + try { + const updated = await updateCatalogColumnMetadata( + databaseId, table.id, active.id, version, + description.trim() || null, generatedDescription.trim() || null, + ); + queryClient.setQueryData(queryKey, (current = []) => current.map((column) => column.id === updated.id ? updated : column)); + setDescription(updated.description ?? ""); + setGeneratedDescription(updated.generatedDescription ?? ""); + setBaseline(JSON.stringify([updated.description ?? "", updated.generatedDescription ?? ""])); + setVersion(updated.version); + setStale(false); + toast.success("Column metadata saved"); + } catch (error) { + if (error instanceof ApiError && error.code === "column_stale") { + await refetch(); + setStale(true); + setStaleBannerOpen(true); + } else toast.error(apiErrorMessage(error)); + } finally { setBusy(false); } + }; + const reloadColumn = async () => { + if (!editingId) return; + setBusy(true); + try { + const result = await refetch(); + const latest = result.data?.find((column) => column.id === editingId); + if (!latest) { closeEditor(); return; } + setDescription(latest.description ?? ""); + setGeneratedDescription(latest.generatedDescription ?? ""); + setBaseline(JSON.stringify([latest.description ?? "", latest.generatedDescription ?? ""])); + setVersion(latest.version); + setStale(false); + setStaleBannerOpen(true); + } catch (error) { toast.error(apiErrorMessage(error)); } finally { setBusy(false); } + }; + + const columns = useMemo[]>(() => [ + { field: "ordinalPosition", headerName: "#", width: 64, maxWidth: 64, filter: "agNumberColumnFilter" }, + { field: "name", headerName: "Name", minWidth: 190, flex: 1, cellClass: "font-mono text-xs" }, + { field: "dataType", headerName: "Type", minWidth: 150, flex: 0.8, cellClass: "font-mono text-xs" }, + { headerName: "Keys", minWidth: 125, width: 125, sortable: false, filter: false, cellRenderer: KeyCell }, + { headerName: "Nullable", minWidth: 100, width: 100, valueGetter: ({ data: row }) => row ? (row.isNullable ? "Yes" : "No") : "" }, + { field: "sourceComment", headerName: "Source comment", minWidth: 220, flex: 1.2, valueFormatter: ({ value }) => value ?? "" }, + { field: "generatedDescription", headerName: "Generated description", minWidth: 230, flex: 1.2, valueFormatter: ({ value }) => value ?? "" }, + { field: "description", headerName: "Description", minWidth: 230, flex: 1.2, valueFormatter: ({ value }) => value ?? "" }, + { colId: "actions", headerName: "", width: 64, maxWidth: 64, pinned: "right", sortable: false, filter: false, resizable: false, cellRenderer: ActionCell }, + ], []); + const context = useMemo(() => ({ canManage, onEdit: edit }), [canManage, data]); + + if (editingId && active) { + return ( +
+ +

{table.name} · column

+

{active.name}

+

Physical schema fields are read-only. Review fields can be edited.

+ {stale ? ( + staleBannerOpen ?

Newer column metadata is available.

Your draft is preserved until you reload.

+ :
Reload the latest value before this metadata can be saved.
+ ) : null} +
+ + + + +