diff --git a/backend/test/workspaces-contracts.test.ts b/backend/test/workspaces-contracts.test.ts index ce4197f8..a51361bc 100644 --- a/backend/test/workspaces-contracts.test.ts +++ b/backend/test/workspaces-contracts.test.ts @@ -1,4 +1,6 @@ import { expect, test } from "vitest"; +import { existsSync, readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; import { parse } from "yaml"; import { buildInstallationContract, renderWorkspaceDocs } from "../src/workspaces/contracts.js"; import { type CanonicalWorkspace, parseWorkspaceYaml } from "../src/workspaces/schema.js"; @@ -170,6 +172,45 @@ llm_policy: })).vector_db).toMatchObject({ database: "vector_database", schema: "vectors" }); }); +test("documents the rendered writer secret-file binding for writer workspaces", () => { + const writerVariable = "THT_WS_PSD_CLINICAL_VECTOR_WRITER_API_KEY_FILE"; + const generated = renderWorkspaceDocs(parseWorkspaceYaml(`workspace: + schema_version: 2 + id: psd-clinical + name: Policlinico San Donato + language: it +dwh: + engine: postgres + database: warehouse + schema: datawarehouse + supported_transports: [postgres_direct] +semantic_index: + vector_store: + engine: pgvector + database: vector_database + schema: vectors + collection: clinical_documents + dimensions: 768 + distance: cosine + supported_transports: [rest_api] + vector_writer: {} + embedding: + provider: ollama_compatible + model: nomic-embed-text-v2-moe + dimensions: 768 +llm_policy: + allowed: [zai/glm-5.2] +`)); + const protocolPath = fileURLToPath(new URL("../../docs/workspace-diagnostic-protocol.md", import.meta.url)); + + expect(generated.markdown).toContain(`\`${writerVariable}\``); + expect(generated.envExample).toContain(`${writerVariable}=`); + expect(existsSync(protocolPath)).toBe(true); + if (existsSync(protocolPath)) { + expect(readFileSync(protocolPath, "utf8")).toContain(writerVariable); + } +}); + function withTransports( dwhTransport: CanonicalWorkspace["dwh"]["supported_transports"][number], vectorTransport: CanonicalWorkspace["semantic_index"]["vector_store"]["supported_transports"][number], diff --git a/docs/superpowers/specs/2026-08-03-git-workspace-registry-design.md b/docs/superpowers/specs/2026-08-03-git-workspace-registry-design.md index f131414c..4cdfe65a 100644 --- a/docs/superpowers/specs/2026-08-03-git-workspace-registry-design.md +++ b/docs/superpowers/specs/2026-08-03-git-workspace-registry-design.md @@ -141,11 +141,34 @@ semantic_index: - pgvector_direct - rest_api - ssh_tunnel + vector_writer: {} # optional; enables a distinct, locally bound reversible diagnostic writer embedding: provider: ollama_compatible model: nomic-embed-text-v2-moe dimensions: 768 +diagnostics: + dwh_rest: + method: POST + path: /rpc/ping + auth: bearer + response: { database: database, schema: schema } + vector_rest: + metadata: + method: GET + path: /vector/metadata + auth: bearer + response: { collection: collection, dimensions: dimensions, distance: distance } + reversible_probe: + method: POST + path: /vector/diagnostic-probe + auth: bearer + embedding: + method: GET + path: /models + auth: none + response: { model: model, dimensions: dimensions } + llm_policy: default: zai/glm-5.2 allowed: @@ -168,6 +191,11 @@ action that supplies the vector database/schema; it must never infer either valu The resulting descriptor is written as schema version 2 and then passes normal operational validation. +The migration also preserves least privilege: `vector_writer` is optional and never inferred from +the reader binding. A v2 descriptor without it is valid and operates reader-only. If it is +declared, its local `VECTOR_WRITER_API_KEY_FILE` is distinct from the reader API-key file and is +used only by the explicitly requested reversible writer diagnostic. + ### 6.2 Semantic-index invariant `semantic_index` is atomic. The vector collection, vector dimensions, distance metric, embedding provider, embedding model, and embedding dimensions describe one index contract. @@ -182,6 +210,21 @@ The following are validation errors: Changing collection, embedding model, dimensions, or metric is presented as replacing or migrating the semantic index, not as an individual user preference. +### 6.3 Declared diagnostic protocol + +Diagnostics are declarative and strict. `dwh_rest` declares the DWH ping method, origin-relative +path, authentication mode, and JSON fields that must equal the canonical DWH database/schema. +`vector_rest.metadata` does the same for collection, dimensions, and distance. `embedding` declares +the model/dimensions response fields. Only `GET` and `POST`, `none`/`bearer`/`x-api-key` +authentication, origin-relative paths without a query or fragment, and identifier-shaped response +field names are accepted. + +`vector_rest.reversible_probe`, when present, is POST-only. It is called with a generated +diagnostic record create request and a matching remove request, with cleanup retried in `finally`. +An upsert-only service cannot be declared as this probe. All ordinary diagnostics remain read-only. +The complete request, response, timeout, reader-only fallback, SSH, and private-CA limitations are +the operator contract in [Workspace diagnostic protocol](../../workspace-diagnostic-protocol.md). + ## 7. Deterministic installation-variable naming The environment namespace is derived from the immutable workspace ID: @@ -230,7 +273,20 @@ THT_WS_PSD_CLINICAL_EMBEDDING_TLS_CA_FILE= The embedding model and dimensions remain in the canonical workspace. -### 7.4 SSH tunnel variables +### 7.4 Optional vector-writer variable + +Only a descriptor declaring `semantic_index.vector_writer: {}` generates this local secret-file +binding. It is never generated for a reader-only workspace: + +```dotenv +THT_WS_PSD_CLINICAL_VECTOR_WRITER_API_KEY_FILE= +``` + +The generated workspace documentation and `.env.example` must render this exact `_FILE` variable +when the optional writer exists. The path must be distinct from +`THT_WS_PSD_CLINICAL_VECTOR_API_KEY_FILE`; neither file's content is rendered. + +### 7.5 SSH tunnel variables For any connector role `` that selects `ssh_tunnel`, ThothII requires: @@ -260,6 +316,11 @@ Direct adapters connect to the configured host and port with the native protocol REST adapters use a base URL, an optional API-key file, TLS validation, and a documented capabilities endpoint. A REST adapter must expose enough metadata to validate schema or collection identity and semantic-index compatibility. +The present diagnostic adapter cannot load a private CA from a REST `*_TLS_CA_FILE` binding. It +therefore refuses that diagnostic rather than weakening certificate verification. Operators must +use a runtime-trusted HTTPS chain, direct/SSH transport with native PostgreSQL CA handling, or a +trusted TLS-termination boundary. + ### 8.3 SSH tunnel SSH adapters verify the remote host against an explicit known-hosts file, open a temporary local tunnel, and pass the resulting endpoint to the corresponding direct adapter. Host-key checking cannot be disabled by the form. @@ -308,6 +369,9 @@ Transport selection is installation-specific because a production server may con A vector write probe is an explicit action. It writes a uniquely named temporary record in a diagnostic namespace or transaction and removes it before returning. It is not part of ordinary save or publish. +When no writer descriptor or distinct local writer file is present, the same workspace remains +reader-only and the write probe is omitted; no reader credential is repurposed for writing. + ## 10. Persistent server and local layout Both production and local Docker deployments use: @@ -450,6 +514,8 @@ Save draft may retain incomplete local form state in the browser. Publish requir - SSH host verification and tunnel opening succeed. - Vector collection, dimensions, metric, and read capability match. - Embedding endpoint exposes the declared model and returns the expected dimensions for a controlled probe. +- A requested writer probe has a declared reversible POST operation, distinct writer credential, + and successful bounded cleanup; otherwise it is omitted without weakening reader validation. A portable workspace can be valid but not activatable on a particular installation. Publish is allowed in that state; starting a new session on that installation is not. diff --git a/docs/workspace-diagnostic-protocol.md b/docs/workspace-diagnostic-protocol.md new file mode 100644 index 00000000..59da0500 --- /dev/null +++ b/docs/workspace-diagnostic-protocol.md @@ -0,0 +1,219 @@ +# Workspace diagnostic protocol + +This is the operator contract for testing a workspace on one ThothII installation. The +Git-shared descriptor declares *what* can be checked; the installation supplies the selected +transport and the local bindings. No secret value, certificate content, SSH key, or response body +belongs in the descriptor, this document, a generated `.env.example`, or diagnostic output. + +## Scope and safety rules + +- The descriptor is schema version 2. Its vector `database` and `schema` are required identity + fields; they are not copied from the DWH, even when both services share PostgreSQL. +- Version 1 descriptors are readable only and have `migration_required` status. An explicit + migration supplies `semantic_index.vector_store.database` and `.schema`, writes version 2, and + must never infer either from `dwh`. +- Each diagnostic is bounded by the configured workspace diagnostic timeout. Redirects are + rejected, response bodies stay inside the adapter, and browser-visible errors are limited to + `binding_missing`, `connector_unavailable`, and `semantic_index_incompatible`. +- A REST path is descriptor-declared, origin-relative, starts with one `/`, and has no query or + fragment. The client may use only the declared method, path, auth mode, and response-field names. +- `auth: none` sends no credential; `auth: bearer` reads a local file and sends + `Authorization: Bearer `; `auth: x-api-key` sends `x-api-key: `. + The file content is never logged or returned. + +## Canonical descriptor additions + +```yaml +semantic_index: + vector_store: + engine: pgvector + database: vector_database + schema: vectors + collection: clinical_documents + dimensions: 768 + distance: cosine + supported_transports: [pgvector_direct, rest_api, ssh_tunnel] + vector_writer: {} # optional: declares a separately bound writer capability + embedding: + provider: ollama_compatible + model: nomic-embed-text-v2-moe + dimensions: 768 + +diagnostics: + dwh_rest: + method: POST + path: /rpc/ping + auth: bearer + response: { database: database, schema: schema } + vector_rest: + metadata: + method: GET + path: /vector/metadata + auth: bearer + response: { collection: collection, dimensions: dimensions, distance: distance } + reversible_probe: + method: POST + path: /vector/diagnostic-probe + auth: bearer + embedding: + method: GET + path: /models + auth: none + response: { model: model, dimensions: dimensions } +``` + +`diagnostics.dwh_rest` requires DWH `rest_api`; `diagnostics.vector_rest` requires vector +`rest_api`. `reversible_probe` is optional, but when present it must be `POST`. Response-map +values are JSON object field names, not values to be put in Git. + +## Installation-local variable contract + +Replace `` with the immutable workspace ID converted to upper case with hyphens changed +to underscores. For example, `psd-clinical` becomes `PSD_CLINICAL`. Set only the variables for the +selected transport. Every `*_FILE` value is an absolute path to a regular, readable file inside an +approved local secret root; it is never the secret itself. + +| Connector and transport | Required local variables | +| --- | --- | +| DWH selection | `THT_WS__DWH_TRANSPORT` | +| DWH `postgres_direct` | `THT_WS__DWH_HOST`, `THT_WS__DWH_PORT`, `THT_WS__DWH_USER`, `THT_WS__DWH_PASSWORD_FILE`; optional `THT_WS__DWH_TLS_CA_FILE` | +| DWH `rest_api` | `THT_WS__DWH_BASE_URL`, `THT_WS__DWH_API_KEY_FILE`; optional `THT_WS__DWH_TLS_CA_FILE` | +| DWH `ssh_tunnel` | `THT_WS__DWH_USER`, `THT_WS__DWH_PASSWORD_FILE`, `THT_WS__DWH_SSH_HOST`, `THT_WS__DWH_SSH_PORT`, `THT_WS__DWH_SSH_USER`, `THT_WS__DWH_SSH_PRIVATE_KEY_FILE`, `THT_WS__DWH_SSH_KNOWN_HOSTS_FILE`, `THT_WS__DWH_SSH_TARGET_HOST`, `THT_WS__DWH_SSH_TARGET_PORT`; optional `THT_WS__DWH_TLS_CA_FILE` | +| Vector selection | `THT_WS__VECTOR_TRANSPORT` | +| Vector `pgvector_direct` | `THT_WS__VECTOR_HOST`, `THT_WS__VECTOR_PORT`, `THT_WS__VECTOR_USER`, `THT_WS__VECTOR_PASSWORD_FILE`; optional `THT_WS__VECTOR_TLS_CA_FILE` | +| Vector `rest_api` | `THT_WS__VECTOR_BASE_URL`, `THT_WS__VECTOR_API_KEY_FILE`; optional `THT_WS__VECTOR_TLS_CA_FILE` | +| Vector `ssh_tunnel` | `THT_WS__VECTOR_USER`, `THT_WS__VECTOR_PASSWORD_FILE`, `THT_WS__VECTOR_SSH_HOST`, `THT_WS__VECTOR_SSH_PORT`, `THT_WS__VECTOR_SSH_USER`, `THT_WS__VECTOR_SSH_PRIVATE_KEY_FILE`, `THT_WS__VECTOR_SSH_KNOWN_HOSTS_FILE`, `THT_WS__VECTOR_SSH_TARGET_HOST`, `THT_WS__VECTOR_SSH_TARGET_PORT`; optional `THT_WS__VECTOR_TLS_CA_FILE` | +| Optional vector writer | `THT_WS__VECTOR_WRITER_API_KEY_FILE` | +| Embedding service | `THT_WS__EMBEDDING_BASE_URL`; optional `THT_WS__EMBEDDING_API_KEY_FILE`, `THT_WS__EMBEDDING_TLS_CA_FILE` | + +For the example workspace, the optional writer name is exactly +`THT_WS_PSD_CLINICAL_VECTOR_WRITER_API_KEY_FILE`. It must resolve to a different local file from +`THT_WS_PSD_CLINICAL_VECTOR_API_KEY_FILE`; a reader key is never substituted for a writer key. + +### Private-CA REST limitation + +The current REST diagnostic adapters use the platform `fetch` implementation and cannot load a +per-request private CA. Therefore a REST diagnostic with any `*_TLS_CA_FILE` binding is refused +rather than silently disabling certificate verification. Use an HTTPS endpoint trusted by the +runtime trust store, use direct or SSH transport where the native PostgreSQL client can validate +the local CA file, or arrange TLS termination at a trusted boundary. This limitation applies to +DWH REST, vector metadata/write REST, and embedding REST diagnostics. + +## DWH diagnostic + +For direct PostgreSQL and SSH-tunnelled PostgreSQL, the diagnostic connects with the declared +`dwh.database`, checks TLS and authentication, then executes exactly: + +```sql +SELECT current_database() AS database, current_schema() AS schema +``` + +Both returned values must equal the descriptor's DWH database and schema. + +For REST, the descriptor above declares the exact ping: + +```text +POST _DWH_BASE_URL>/rpc/ping +Authorization: Bearer +``` + +It has no request body. A 2xx response must be a JSON object whose declared `database` and +`schema` fields equal `dwh.database` and `dwh.schema`. For the sample response map, that is: + +```json +{ "database": "warehouse", "schema": "datawarehouse" } +``` + +The values are illustrative resource identities, not credentials. A different response-field map +is valid only when the descriptor declares it. + +## Vector metadata diagnostic + +Direct and SSH vector checks connect to the *vector* `database` and `schema`, not the DWH +identity. They inspect the declared collection's vector column and index and must find the exact +collection, integer `dimensions`, and `distance` (`cosine`, `l2`, or `inner_product`) declared in +`semantic_index.vector_store`. + +For REST, the exact descriptor-declared request is, for example: + +```text +GET _VECTOR_BASE_URL>/vector/metadata +Authorization: Bearer +``` + +It has no request body. A 2xx JSON object must supply the declared `collection`, `dimensions`, and +`distance` fields. All three values must exactly match the vector-store contract; an integer +dimension is required. Metadata from a similarly named collection, a different metric, or a +different dimension makes the semantic index incompatible. + +## Reversible vector writer probe + +Ordinary validation is reader-only. A write probe runs only when all of the following are true: + +1. The operator explicitly requests it. +2. The descriptor has `semantic_index.vector_writer: {}`. +3. The descriptor declares `diagnostics.vector_rest.reversible_probe`. +4. The selected vector transport is `rest_api`. +5. `THT_WS__VECTOR_WRITER_API_KEY_FILE` exists locally and is distinct from the reader + API-key file. + +The probe uses the declared `POST` endpoint twice, with the same generated ID and the writer key: + +```json +{ "operation": "create", "id": "diagnostic:", "collection": "", "dimensions": 768 } +``` + +then: + +```json +{ "operation": "remove", "id": "diagnostic:", "collection": "" } +``` + +Both requests require a 2xx response. Cleanup is attempted in `finally`, including after a write +timeout or error. The endpoint must implement both operations as a bounded, reversible diagnostic +operation; an upsert-only endpoint is prohibited. It must not retain, index, or expose diagnostic +records. If the writer capability or its local binding is absent, validation remains reader-only +and no write request is sent. + +## Embedding dimensions diagnostic + +The embedding request is descriptor-declared, for example: + +```text +GET _EMBEDDING_BASE_URL>/models +``` + +It has no body and uses the declared authentication mode. A 2xx JSON object must contain the +declared model field equal to `semantic_index.embedding.model` and a declared dimensions field that +is an integer. That integer must equal both `semantic_index.embedding.dimensions` and +`semantic_index.vector_store.dimensions`. + +## SSH host verification and tunnel lifecycle + +For either DWH or vector `ssh_tunnel`, the known-hosts file is mandatory and is verified before a +connection is accepted. The tunnel is a short-lived loopback forward for the diagnostic only. The +effective OpenSSH constraints are: + +```text +-N -v +-o BatchMode=yes +-o ExitOnForwardFailure=yes +-o StrictHostKeyChecking=yes +-o UserKnownHostsFile=_SSH_KNOWN_HOSTS_FILE +-i _SSH_PRIVATE_KEY_FILE +-p _SSH_PORT +-L 127.0.0.1::_SSH_TARGET_HOST:_SSH_TARGET_PORT +_SSH_USER@_SSH_HOST +``` + +The local listener is `127.0.0.1` only. The process is terminated in cleanup after the direct +probe, on timeout, or on failure. There is no accept-new mode, no disabled host-key checking, and +no persistent forwarding. + +## Reader-only fallback + +A workspace may be fully valid in Git but non-activatable locally when a required reader binding, +secret file, host verification, TLS check, or declared diagnostic fails. That state does not alter +the shared descriptor and does not permit a new session on that installation. It may still be +published and activated elsewhere with valid local bindings. Missing optional writer capability is +not a reader failure: it leaves the workspace in reader-only mode and suppresses the writer probe.