diff --git a/docs/install/examples/connector-secrets.workspace-registry.yaml b/docs/install/examples/connector-secrets.workspace-registry.yaml new file mode 100644 index 00000000..bb26ff8a --- /dev/null +++ b/docs/install/examples/connector-secrets.workspace-registry.yaml @@ -0,0 +1,16 @@ +# Reviewed direct PostgreSQL/pgvector connector-secret override for workspace-bindings.env.example. +# Source variables are absolute host paths. Add only matching entries for the selected transport; +# targets must equal the corresponding THT_WS_*_FILE paths in the bindings env file. +services: + core: + secrets: + - source: psd_clinical_dwh_password + target: psd-clinical-dwh-password + - source: psd_clinical_vector_password + target: psd-clinical-vector-password + +secrets: + psd_clinical_dwh_password: + file: ${THT_WS_PSD_CLINICAL_DWH_PASSWORD_SOURCE:?set THT_WS_PSD_CLINICAL_DWH_PASSWORD_SOURCE} + psd_clinical_vector_password: + file: ${THT_WS_PSD_CLINICAL_VECTOR_PASSWORD_SOURCE:?set THT_WS_PSD_CLINICAL_VECTOR_PASSWORD_SOURCE} diff --git a/docs/install/examples/local-compose.workspace-registry.yaml b/docs/install/examples/local-compose.workspace-registry.yaml index db1e2107..460c91ed 100644 --- a/docs/install/examples/local-compose.workspace-registry.yaml +++ b/docs/install/examples/local-compose.workspace-registry.yaml @@ -8,6 +8,9 @@ services: build: context: ${THT_SOURCE_ROOT:?set THT_SOURCE_ROOT to the absolute ThothII source checkout} dockerfile: docker/core.Dockerfile + env_file: + - path: ${THT_WORKSPACE_BINDINGS_ENV_FILE:?set THT_WORKSPACE_BINDINGS_ENV_FILE to an absolute THT_WS bindings file} + required: true environment: HOST: 0.0.0.0 PORT: "8787" diff --git a/docs/install/examples/server-compose.workspace-registry.yaml b/docs/install/examples/server-compose.workspace-registry.yaml index 0dff8069..996800c9 100644 --- a/docs/install/examples/server-compose.workspace-registry.yaml +++ b/docs/install/examples/server-compose.workspace-registry.yaml @@ -8,6 +8,9 @@ services: build: context: ${THT_SOURCE_ROOT:?set THT_SOURCE_ROOT to the absolute ThothII source checkout} dockerfile: docker/core.Dockerfile + env_file: + - path: ${THT_WORKSPACE_BINDINGS_ENV_FILE:?set THT_WORKSPACE_BINDINGS_ENV_FILE to an absolute THT_WS bindings file} + required: true environment: HOST: 0.0.0.0 PORT: "8787" diff --git a/docs/install/examples/workspace-bindings.env.example b/docs/install/examples/workspace-bindings.env.example new file mode 100644 index 00000000..544d29ee --- /dev/null +++ b/docs/install/examples/workspace-bindings.env.example @@ -0,0 +1,13 @@ +# Copy to an untracked operator file. This file contains only non-secret THT_WS_* bindings. +# Every *_FILE value is a container path supplied by a reviewed connector-secret override. +THT_WS_PSD_CLINICAL_DWH_TRANSPORT=postgres_direct +THT_WS_PSD_CLINICAL_DWH_HOST=dwh.internal.example +THT_WS_PSD_CLINICAL_DWH_PORT=5432 +THT_WS_PSD_CLINICAL_DWH_USER=thoth_reader +THT_WS_PSD_CLINICAL_DWH_PASSWORD_FILE=/run/secrets/psd-clinical-dwh-password +THT_WS_PSD_CLINICAL_VECTOR_TRANSPORT=pgvector_direct +THT_WS_PSD_CLINICAL_VECTOR_HOST=vector.internal.example +THT_WS_PSD_CLINICAL_VECTOR_PORT=5432 +THT_WS_PSD_CLINICAL_VECTOR_USER=thoth_vector_reader +THT_WS_PSD_CLINICAL_VECTOR_PASSWORD_FILE=/run/secrets/psd-clinical-vector-password +THT_WS_PSD_CLINICAL_EMBEDDING_BASE_URL=https://embeddings.internal.example diff --git a/docs/install/local-workspace-registry.md b/docs/install/local-workspace-registry.md index 02ec06b4..03a1223e 100644 --- a/docs/install/local-workspace-registry.md +++ b/docs/install/local-workspace-registry.md @@ -55,7 +55,8 @@ and branch are non-secret; every `*_FILE` is a local path whose content never en | Location | Contains | Never contains | | --- | --- | --- | | Git workspace repository | schema v2 YAML, generated binding names, LLM policy, model/index identity | installation hostnames, keys, passwords, certificates, SSH keys | -| local `.env` | remote, branch, installation ID, selected transport and endpoints | secret contents | +| local `.env` | remote, branch, installation ID, `THT_WORKSPACE_BINDINGS_ENV_FILE`, and secret source paths | secret contents or `THT_WS_*` values | +| workspace bindings env file | only `THT_WS_*` transport, endpoint, user, and `/run/secrets/...` path bindings | secret contents or unrelated application settings | | local secret directory | Git credentials/key, known hosts, CA, connector secret files | a copied registry checkout | | Docker volumes | registry checkout/snapshots/state/locks and local data | host-only secret source files | @@ -69,14 +70,20 @@ locks/ # short-lived publish locks ``` Installation variables are deterministic: `psd-clinical` becomes `PSD_CLINICAL`, and every name -is `THT_WS___`. Credentials and certificates use `*_FILE` path variables. +is `THT_WS___`. Copy +[the bindings env example](examples/workspace-bindings.env.example) to an untracked operator file +and set its absolute path as `THT_WORKSPACE_BINDINGS_ENV_FILE`. It is loaded only into `core`. +Credentials and certificates use `*_FILE` path variables that must point inside `/run/secrets`. If declared, `THT_WS_PSD_CLINICAL_VECTOR_WRITER_API_KEY_FILE` is distinct from the vector reader file; a reader credential is never repurposed for writing. ## Direct PostgreSQL, REST, and SSH tunnel bindings -Set only fields for the selected transport. Canonical YAML keeps database/schema/collection, -distance, embedding model, and dimensions shared in Git. +Set only fields for the selected transport in the dedicated bindings env file. Canonical YAML keeps +database/schema/collection, distance, embedding model, and dimensions shared in Git. Copy and +review [the connector-secret override](examples/connector-secrets.workspace-registry.yaml) for the +selected transport: every `*_FILE=/run/secrets/` binding needs one matching Docker secret +target and one host-only `*_SOURCE` path in operator `.env`. ```dotenv # Direct PostgreSQL and pgvector @@ -125,19 +132,21 @@ rather than weakening TLS; use runtime-trusted HTTPS or verified direct/SSH nati Copy [the local Compose example](examples/local-compose.workspace-registry.yaml) and exactly one selected [SSH Git override](examples/git-ssh.workspace-registry.yaml) or [HTTPS Git override](examples/git-https.workspace-registry.yaml) -into an untracked operator directory. Set `THT_SOURCE_ROOT` in its `.env` to the absolute source -checkout path; this keeps the copied Compose file buildable. Create only the secret files used by -the selected override, then render it before start. +plus [the bindings env example](examples/workspace-bindings.env.example) and a reviewed +[connector-secret override](examples/connector-secrets.workspace-registry.yaml) into an untracked +operator directory. Set `THT_SOURCE_ROOT` and the absolute `THT_WORKSPACE_BINDINGS_ENV_FILE` in +its `.env`; this keeps the copied Compose file buildable and confines `THT_WS_*` values to `core`. +Create only the host secret files named by the selected Git/connector override, then render it. ```sh -THT_SOURCE_ROOT="$(pwd -P)" docker compose -f docs/install/examples/local-compose.workspace-registry.yaml config --quiet +THT_SOURCE_ROOT="$(pwd -P)" THT_WORKSPACE_BINDINGS_ENV_FILE="$(pwd -P)/docs/install/examples/workspace-bindings.env.example" docker compose -f docs/install/examples/local-compose.workspace-registry.yaml config --quiet ``` From the operator directory: ```sh -docker compose -f compose.workspace-registry.yaml -f git-ssh.workspace-registry.yaml up --build -d +docker compose -f compose.workspace-registry.yaml -f git-ssh.workspace-registry.yaml -f connector-secrets.workspace-registry.yaml up --build -d curl --fail --silent http://127.0.0.1:8787/health curl --fail --silent http://127.0.0.1:8787/workspace-registry/status curl --fail --silent http://127.0.0.1:8787/workspaces diff --git a/docs/install/server-workspace-registry.md b/docs/install/server-workspace-registry.md index f13717ae..f7be498c 100644 --- a/docs/install/server-workspace-registry.md +++ b/docs/install/server-workspace-registry.md @@ -61,8 +61,10 @@ restarting `core`, and performing pull/status; never put the material in an envi Git describes workspace schema, immutable ID, DWH/vector identity, semantic-index dimensions and distance, embedding contract, and LLM policy. The installation supplies remote/branch/installation -ID, transport, endpoints, users, ports, and `THT_WS_*` bindings. Secret contents are only in files, -never the values stored in Git or browser-local drafts. +ID and one absolute `THT_WORKSPACE_BINDINGS_ENV_FILE` containing only `THT_WS_*` transport, +endpoint, user, and `/run/secrets/...` path bindings. The base Compose loads that file only into +`core`. Secret contents are only in host files, never the values stored in Git or browser-local +drafts. The runtime registry layout is persistent and must be backed up together: @@ -76,6 +78,10 @@ The runtime registry layout is persistent and must be backed up together: Variable names derive from the immutable ID: `psd-clinical` becomes `PSD_CLINICAL`, producing `THT_WS_PSD_CLINICAL_DWH_PASSWORD_FILE`. A declared vector writer uses the distinct `THT_WS_PSD_CLINICAL_VECTOR_WRITER_API_KEY_FILE`; a reader file is never a writer substitute. +Copy [the bindings env example](examples/workspace-bindings.env.example) to the protected operator +directory. Every path-valued `*_FILE` entry must be supplied by a reviewed +[connector-secret override](examples/connector-secrets.workspace-registry.yaml) with a matching +Docker secret target below `/run/secrets` and an absolute host-only `*_SOURCE` path. ## Direct PostgreSQL, REST, and SSH tunnel bindings @@ -132,7 +138,9 @@ Copy [the server Compose example](examples/server-compose.workspace-registry.yam selected Git override to the protected operator directory. Set `THT_SOURCE_ROOT` to the absolute ThothII checkout; a copied file cannot use a relative build context. Copy `deploy/workspaces/server-sessions.yaml.example` into that operator directory, review it, then set -the absolute `THT_SERVER_WORKSPACE_CONFIG` path. The same `.env` must set +the absolute `THT_SERVER_WORKSPACE_CONFIG` path. Copy the bindings env example and a reviewed +connector-secret override, then set absolute `THT_WORKSPACE_BINDINGS_ENV_FILE` and connector +`*_SOURCE` paths. The same `.env` must set `THT_SESSION_DB_HOST`, `THT_SESSION_DB_NAME`, `THT_SESSION_RUNTIME_USER`, `THT_SESSION_RUNTIME_PASSWORD_SOURCE`, and `THT_SESSION_CA_SOURCE`; the base Compose file wires `postgres`, `verify-full`, and the two Docker secret mount paths. This is the public server profile, @@ -148,7 +156,7 @@ forwards the trusted identity expected by `AUTH_MODE=upstream`; it is the only p From a trusted maintenance shell: ```sh -docker compose -f compose.workspace-registry.yaml -f git-ssh.workspace-registry.yaml up --build -d +docker compose -f compose.workspace-registry.yaml -f git-ssh.workspace-registry.yaml -f connector-secrets.workspace-registry.yaml up --build -d docker compose -f compose.workspace-registry.yaml exec -T core curl --fail --silent http://127.0.0.1:8787/health docker compose -f compose.workspace-registry.yaml exec -T core curl --fail --silent http://127.0.0.1:8787/workspace-registry/status ``` diff --git a/scripts/test-verify-workspace-install-docs.sh b/scripts/test-verify-workspace-install-docs.sh index 863e4bfd..c74c880d 100755 --- a/scripts/test-verify-workspace-install-docs.sh +++ b/scripts/test-verify-workspace-install-docs.sh @@ -13,6 +13,8 @@ for fixture in \ "copied server PostgreSQL/TLS fixture" \ "copied HTTPS Git override fixture" \ "copied SSH Git override fixture" \ + "copied connector binding/secret fixture" \ + "core process sees connector bindings and secret files" \ "non-path secret-file fixture rejected"; do grep -Fqx "$fixture passed" "$output" >/dev/null || { echo "missing fixture verification: $fixture" >&2 diff --git a/scripts/verify-workspace-install-docs.sh b/scripts/verify-workspace-install-docs.sh index beee4bb2..24f0d9e2 100755 --- a/scripts/verify-workspace-install-docs.sh +++ b/scripts/verify-workspace-install-docs.sh @@ -59,16 +59,68 @@ compose_fixture() { echo "$name passed" } +prepare_binding_fixture() { + local directory="$1" + cp "$root/docs/install/examples/workspace-bindings.env.example" "$directory/workspace-bindings.env" + printf 'THT_WORKSPACE_BINDINGS_ENV_FILE=%s\n' "$directory/workspace-bindings.env" >>"$directory/.env" +} + +verify_connector_fixture() { + local directory="$1" rendered project + project="thoth-install-connector-fixture-$$" + rendered="$( + cd "$directory" + docker compose --env-file .env -f compose.workspace-registry.yaml -f connector-secrets.yaml config + )" + for expected in \ + 'THT_WS_PSD_CLINICAL_DWH_TRANSPORT: postgres_direct' \ + 'THT_WS_PSD_CLINICAL_DWH_PASSWORD_FILE: /run/secrets/psd-clinical-dwh-password' \ + 'THT_WS_PSD_CLINICAL_VECTOR_PASSWORD_FILE: /run/secrets/psd-clinical-vector-password' \ + 'target: psd-clinical-dwh-password' \ + 'target: psd-clinical-vector-password'; do + grep -Fq "$expected" <<<"$rendered" || { + echo "connector fixture does not give core required binding or secret target: $expected" >&2 + return 1 + } + done + echo "copied connector binding/secret fixture passed" + if ! ( + cd "$directory" + docker compose --project-name "$project" --env-file .env -f compose.workspace-registry.yaml -f connector-secrets.yaml \ + run --rm --no-deps --build --entrypoint sh core -c ' + test "$THT_WS_PSD_CLINICAL_DWH_TRANSPORT" = postgres_direct + test "$THT_WS_PSD_CLINICAL_DWH_PASSWORD_FILE" = /run/secrets/psd-clinical-dwh-password + test "$THT_WS_PSD_CLINICAL_VECTOR_PASSWORD_FILE" = /run/secrets/psd-clinical-vector-password + test -f "$THT_WS_PSD_CLINICAL_DWH_PASSWORD_FILE" + test -f "$THT_WS_PSD_CLINICAL_VECTOR_PASSWORD_FILE" + ' + ); then + ( + cd "$directory" + docker compose --project-name "$project" --env-file .env -f compose.workspace-registry.yaml -f connector-secrets.yaml \ + down --volumes --remove-orphans + ) || true + return 1 + fi + ( + cd "$directory" + docker compose --project-name "$project" --env-file .env -f compose.workspace-registry.yaml -f connector-secrets.yaml \ + down --volumes --remove-orphans + ) + echo "core process sees connector bindings and secret files passed" +} + verify_copied_operator_fixtures() { - local fixture_root local_dir server_dir https_dir ssh_dir + local fixture_root local_dir server_dir https_dir ssh_dir connector_dir fixture_root="$(mktemp -d "${TMPDIR:-/tmp}/thoth-install-fixtures.XXXXXX")" trap 'rm -rf "$fixture_root"' RETURN local_dir="$fixture_root/local"; server_dir="$fixture_root/server" - https_dir="$fixture_root/https"; ssh_dir="$fixture_root/ssh" - mkdir -p "$local_dir" "$server_dir" "$https_dir" "$ssh_dir" + https_dir="$fixture_root/https"; ssh_dir="$fixture_root/ssh"; connector_dir="$fixture_root/connector" + mkdir -p "$local_dir" "$server_dir" "$https_dir" "$ssh_dir" "$connector_dir" cp "$root/docs/install/examples/local-compose.workspace-registry.yaml" "$local_dir/compose.workspace-registry.yaml" printf 'THT_SOURCE_ROOT=%s\n' "$root" >"$local_dir/.env" + prepare_binding_fixture "$local_dir" compose_fixture "copied local base fixture" "$local_dir" -f compose.workspace-registry.yaml cp "$root/docs/install/examples/server-compose.workspace-registry.yaml" "$server_dir/compose.workspace-registry.yaml" @@ -82,6 +134,7 @@ verify_copied_operator_fixtures() { 'THT_SESSION_RUNTIME_USER=thoth_sessions_app' \ "THT_SESSION_RUNTIME_PASSWORD_SOURCE=$server_dir/session-runtime-password" \ "THT_SESSION_CA_SOURCE=$server_dir/session-ca.pem" >"$server_dir/.env" + prepare_binding_fixture "$server_dir" compose_fixture "copied server PostgreSQL/TLS fixture" "$server_dir" -f compose.workspace-registry.yaml cp "$root/docs/install/examples/local-compose.workspace-registry.yaml" "$https_dir/compose.workspace-registry.yaml" @@ -91,6 +144,7 @@ verify_copied_operator_fixtures() { "THT_SOURCE_ROOT=$root" \ "THT_WORKSPACE_GIT_CREDENTIALS_FILE=$https_dir/git-credentials" \ "THT_WORKSPACE_GIT_CA_FILE=$https_dir/git-ca.pem" >"$https_dir/.env" + prepare_binding_fixture "$https_dir" compose_fixture "copied HTTPS Git override fixture" "$https_dir" -f compose.workspace-registry.yaml -f git-https.yaml cp "$root/docs/install/examples/local-compose.workspace-registry.yaml" "$ssh_dir/compose.workspace-registry.yaml" @@ -100,8 +154,19 @@ verify_copied_operator_fixtures() { "THT_SOURCE_ROOT=$root" \ "THT_WORKSPACE_GIT_SSH_KEY_FILE=$ssh_dir/git-ssh-key" \ "THT_WORKSPACE_GIT_KNOWN_HOSTS_FILE=$ssh_dir/git-known-hosts" >"$ssh_dir/.env" + prepare_binding_fixture "$ssh_dir" compose_fixture "copied SSH Git override fixture" "$ssh_dir" -f compose.workspace-registry.yaml -f git-ssh.yaml + cp "$root/docs/install/examples/local-compose.workspace-registry.yaml" "$connector_dir/compose.workspace-registry.yaml" + cp "$root/docs/install/examples/connector-secrets.workspace-registry.yaml" "$connector_dir/connector-secrets.yaml" + : >"$connector_dir/dwh-password"; : >"$connector_dir/vector-password" + printf '%s\n' \ + "THT_SOURCE_ROOT=$root" \ + "THT_WS_PSD_CLINICAL_DWH_PASSWORD_SOURCE=$connector_dir/dwh-password" \ + "THT_WS_PSD_CLINICAL_VECTOR_PASSWORD_SOURCE=$connector_dir/vector-password" >"$connector_dir/.env" + prepare_binding_fixture "$connector_dir" + verify_connector_fixture "$connector_dir" + printf 'THT_WS_EXAMPLE_DWH_PASSWORD_FILE=not-a-path\n' >"$fixture_root/non-path-secret.env" if verify_secret_file_values "$fixture_root/non-path-secret.env" >/dev/null 2>&1; then echo "non-path secret-file fixture was accepted" >&2 @@ -186,6 +251,8 @@ verify_secret_file_values "$manual" verify_secret_file_values "$example" verify_secret_file_values "$root/docs/install/examples/git-https.workspace-registry.yaml" verify_secret_file_values "$root/docs/install/examples/git-ssh.workspace-registry.yaml" +verify_secret_file_values "$root/docs/install/examples/workspace-bindings.env.example" +verify_secret_file_values "$root/docs/install/examples/connector-secrets.workspace-registry.yaml" verify_server_public_contract commands="$(mktemp "${TMPDIR:-/tmp}/thoth-install-docs.XXXXXX")"