diff --git a/docs/install/examples/connector-secrets.workspace-registry.yaml b/docs/install/examples/connector-secrets.workspace-registry.yaml deleted file mode 100644 index bb26ff8a..00000000 --- a/docs/install/examples/connector-secrets.workspace-registry.yaml +++ /dev/null @@ -1,16 +0,0 @@ -# Reviewed direct PostgreSQL/pgvector connector-secret override for workspace-bindings.env.example. -# Source variables are absolute host paths. Add only matching entries for the selected transport; -# targets must equal the corresponding THT_WS_*_FILE paths in the bindings env file. -services: - core: - secrets: - - source: psd_clinical_dwh_password - target: psd-clinical-dwh-password - - source: psd_clinical_vector_password - target: psd-clinical-vector-password - -secrets: - psd_clinical_dwh_password: - file: ${THT_WS_PSD_CLINICAL_DWH_PASSWORD_SOURCE:?set THT_WS_PSD_CLINICAL_DWH_PASSWORD_SOURCE} - psd_clinical_vector_password: - file: ${THT_WS_PSD_CLINICAL_VECTOR_PASSWORD_SOURCE:?set THT_WS_PSD_CLINICAL_VECTOR_PASSWORD_SOURCE} diff --git a/docs/install/examples/workspace-bindings.env.example b/docs/install/examples/workspace-bindings.env.example index 544d29ee..fe511fc6 100644 --- a/docs/install/examples/workspace-bindings.env.example +++ b/docs/install/examples/workspace-bindings.env.example @@ -1,5 +1,5 @@ # Copy to an untracked operator file. This file contains only non-secret THT_WS_* bindings. -# Every *_FILE value is a container path supplied by a reviewed connector-secret override. +# Every *_FILE value is a container path supplied by the generated local connector override. THT_WS_PSD_CLINICAL_DWH_TRANSPORT=postgres_direct THT_WS_PSD_CLINICAL_DWH_HOST=dwh.internal.example THT_WS_PSD_CLINICAL_DWH_PORT=5432 diff --git a/docs/install/local-workspace-registry.md b/docs/install/local-workspace-registry.md index 989753dc..54fe2abd 100644 --- a/docs/install/local-workspace-registry.md +++ b/docs/install/local-workspace-registry.md @@ -80,10 +80,10 @@ file; a reader credential is never repurposed for writing. ## Direct PostgreSQL, REST, and SSH tunnel bindings Set only fields for the selected transport in the dedicated bindings env file. Canonical YAML keeps -database/schema/collection, distance, embedding model, and dimensions shared in Git. Copy and -review [the connector-secret override](examples/connector-secrets.workspace-registry.yaml) for the -selected transport: every `*_FILE=/run/secrets/` binding needs one matching Docker secret -target and one host-only `*_SOURCE` path in operator `.env`. +database/schema/collection, distance, embedding model, and dimensions shared in Git. Every +`*_FILE=/run/secrets/` binding needs one matching host-only `*_SOURCE` path in operator +`.env`. Generate the untracked connector override from those two files during bootstrap; do not +copy or maintain a workspace-specific Compose override. ```dotenv # Direct PostgreSQL and pgvector @@ -134,23 +134,26 @@ before creating sessions. Git pull/push over SSH remains fully supported and is ## Bootstrap, first pull, and diagnostics -Copy [the local Compose example](examples/local-compose.workspace-registry.yaml) and exactly one -selected [SSH Git override](examples/git-ssh.workspace-registry.yaml) or [HTTPS Git override](examples/git-https.workspace-registry.yaml) -plus [the bindings env example](examples/workspace-bindings.env.example) and a reviewed -[connector-secret override](examples/connector-secrets.workspace-registry.yaml) into an untracked -operator directory. Set `THT_SOURCE_ROOT` and the absolute `THT_WORKSPACE_BINDINGS_ENV_FILE` in -its `.env`; this keeps the copied Compose file buildable and confines `THT_WS_*` values to `core`. -Create only the host secret files named by the selected Git/connector override, then render it. +Copy [the local Compose example](examples/local-compose.workspace-registry.yaml), exactly one +selected [SSH Git override](examples/git-ssh.workspace-registry.yaml) or [HTTPS Git override](examples/git-https.workspace-registry.yaml), +and [the bindings env example](examples/workspace-bindings.env.example) into an untracked operator +directory. Set `THT_SOURCE_ROOT` and the absolute `THT_WORKSPACE_BINDINGS_ENV_FILE` in its `.env`; +this keeps the copied Compose file buildable and confines `THT_WS_*` values to `core`. Create the +host secret files named by the selected Git transport and every declared connector `*_SOURCE`, then +generate the connector override and render through the preflight wrapper. The wrapper is required: +it rejects unsafe source paths and a combined SSH+HTTPS Git selection before Compose runs. - ```sh -THT_SOURCE_ROOT="$(pwd -P)" THT_WORKSPACE_BINDINGS_ENV_FILE="$(pwd -P)/docs/install/examples/workspace-bindings.env.example" docker compose -f docs/install/examples/local-compose.workspace-registry.yaml config --quiet +"$THT_SOURCE_ROOT/scripts/generate-connector-secrets-override.sh" --bindings-env "$THT_WORKSPACE_BINDINGS_ENV_FILE" --operator-env .env --output connector-secrets.local.yaml +"$THT_SOURCE_ROOT/scripts/compose-with-preflight.sh" --env-file .env \ + -f compose.workspace-registry.yaml -f git-ssh.workspace-registry.yaml -f connector-secrets.local.yaml config --quiet ``` From the operator directory: ```sh -docker compose -f compose.workspace-registry.yaml -f git-ssh.workspace-registry.yaml -f connector-secrets.workspace-registry.yaml up --build -d +"$THT_SOURCE_ROOT/scripts/compose-with-preflight.sh" --env-file .env \ + -f compose.workspace-registry.yaml -f git-ssh.workspace-registry.yaml -f connector-secrets.local.yaml up --build -d curl --fail --silent http://127.0.0.1:8787/health curl --fail --silent http://127.0.0.1:8787/workspace-registry/status curl --fail --silent http://127.0.0.1:8787/workspaces diff --git a/docs/install/server-workspace-registry.md b/docs/install/server-workspace-registry.md index 48b28f19..f1f44e4a 100644 --- a/docs/install/server-workspace-registry.md +++ b/docs/install/server-workspace-registry.md @@ -55,7 +55,7 @@ mounts neither transport; add exactly one [HTTPS override](examples/git-https.wo or [SSH override](examples/git-ssh.workspace-registry.yaml). Strict host-key checking stays enabled and Git stderr is not exposed by the API. Rotate by atomically replacing the secret file, restarting `core`, and performing pull/status; never put the material in an environment variable or -`docker compose config` output. +rendered Compose output. ## Shared Git values, local bindings, and secret files @@ -79,9 +79,9 @@ Variable names derive from the immutable ID: `psd-clinical` becomes `PSD_CLINICA `THT_WS_PSD_CLINICAL_DWH_PASSWORD_FILE`. A declared vector writer uses the distinct `THT_WS_PSD_CLINICAL_VECTOR_WRITER_API_KEY_FILE`; a reader file is never a writer substitute. Copy [the bindings env example](examples/workspace-bindings.env.example) to the protected operator -directory. Every path-valued `*_FILE` entry must be supplied by a reviewed -[connector-secret override](examples/connector-secrets.workspace-registry.yaml) with a matching -Docker secret target below `/run/secrets` and an absolute host-only `*_SOURCE` path. +directory. Every path-valued `*_FILE` entry needs an absolute host-only `*_SOURCE` path. Generate +the untracked connector override from those files during bootstrap; do not copy or maintain a +workspace-specific Compose override. ## Direct PostgreSQL, REST, and SSH tunnel bindings @@ -142,27 +142,25 @@ Copy [the server Compose example](examples/server-compose.workspace-registry.yam selected Git override to the protected operator directory. Set `THT_SOURCE_ROOT` to the absolute ThothII checkout; a copied file cannot use a relative build context. Copy `deploy/workspaces/server-sessions.yaml.example` into that operator directory, review it, then set -the absolute `THT_SERVER_WORKSPACE_CONFIG` path. Copy the bindings env example and a reviewed -connector-secret override, then set absolute `THT_WORKSPACE_BINDINGS_ENV_FILE` and connector -`*_SOURCE` paths. The same `.env` must set +the absolute `THT_SERVER_WORKSPACE_CONFIG` path. Copy the bindings env example, then set absolute +`THT_WORKSPACE_BINDINGS_ENV_FILE` and connector `*_SOURCE` paths. The same `.env` must set `THT_SESSION_DB_HOST`, `THT_SESSION_DB_NAME`, `THT_SESSION_RUNTIME_USER`, `THT_SESSION_RUNTIME_PASSWORD_SOURCE`, and `THT_SESSION_CA_SOURCE`; the base Compose file wires `postgres`, `verify-full`, and the two Docker secret mount paths. This is the public server profile, not a filesystem-session fallback. - -```sh -./scripts/verify-workspace-install-docs.sh --fixtures-only -``` - Configure the portal proxy so the frontend and `/api` share one origin. It authenticates first and forwards the trusted identity expected by `AUTH_MODE=upstream`; it is the only public listener. From a trusted maintenance shell: ```sh -docker compose -f compose.workspace-registry.yaml -f git-ssh.workspace-registry.yaml -f connector-secrets.workspace-registry.yaml up --build -d -docker compose -f compose.workspace-registry.yaml exec -T core curl --fail --silent http://127.0.0.1:8787/health -docker compose -f compose.workspace-registry.yaml exec -T core curl --fail --silent http://127.0.0.1:8787/workspace-registry/status +"$THT_SOURCE_ROOT/scripts/generate-connector-secrets-override.sh" --bindings-env "$THT_WORKSPACE_BINDINGS_ENV_FILE" --operator-env .env --output connector-secrets.local.yaml +"$THT_SOURCE_ROOT/scripts/compose-with-preflight.sh" --env-file .env \ + -f compose.workspace-registry.yaml -f git-ssh.workspace-registry.yaml -f connector-secrets.local.yaml up --build -d +"$THT_SOURCE_ROOT/scripts/compose-with-preflight.sh" --env-file .env \ + -f compose.workspace-registry.yaml -f git-ssh.workspace-registry.yaml -f connector-secrets.local.yaml exec -T core curl --fail --silent http://127.0.0.1:8787/health +"$THT_SOURCE_ROOT/scripts/compose-with-preflight.sh" --env-file .env \ + -f compose.workspace-registry.yaml -f git-ssh.workspace-registry.yaml -f connector-secrets.local.yaml exec -T core curl --fail --silent http://127.0.0.1:8787/workspace-registry/status ``` `/health` is liveness. Registry status verifies branch/head/degraded state and the active validated diff --git a/scripts/test-verify-workspace-install-docs.sh b/scripts/test-verify-workspace-install-docs.sh index 57ee6f51..13810f90 100755 --- a/scripts/test-verify-workspace-install-docs.sh +++ b/scripts/test-verify-workspace-install-docs.sh @@ -9,6 +9,8 @@ trap 'rm -f "$output"' EXIT HUP INT TERM "$root/scripts/verify-workspace-install-docs.sh" --fixtures-only >"$output" for fixture in \ + "local manual requires generated connector override and Compose preflight" \ + "server manual requires generated connector override and Compose preflight" \ "copied local base fixture" \ "copied server PostgreSQL/TLS fixture" \ "copied HTTPS Git override fixture" \ @@ -25,3 +27,10 @@ for fixture in \ exit 1 } done + +if rg -n 'connector-secrets\.workspace-registry|docker compose' \ + "$root/docs/install/local-workspace-registry.md" \ + "$root/docs/install/server-workspace-registry.md"; then + echo "installation manuals still document a bypassed Compose or copied connector override path" >&2 + exit 1 +fi diff --git a/scripts/verify-workspace-install-docs.sh b/scripts/verify-workspace-install-docs.sh index 56a69e59..19e7d64a 100755 --- a/scripts/verify-workspace-install-docs.sh +++ b/scripts/verify-workspace-install-docs.sh @@ -60,6 +60,26 @@ verify_server_public_contract() { done } +verify_manual_supported_path() { + local profile="$1" manual="$2" + local generator='"$THT_SOURCE_ROOT/scripts/generate-connector-secrets-override.sh" --bindings-env "$THT_WORKSPACE_BINDINGS_ENV_FILE" --operator-env .env --output connector-secrets.local.yaml' + local wrapper='"$THT_SOURCE_ROOT/scripts/compose-with-preflight.sh" --env-file .env' + + grep -Fq "$generator" "$manual" || { + echo "$profile manual does not document the connector override generator" >&2 + return 1 + } + grep -Fq "$wrapper" "$manual" || { + echo "$profile manual does not document the Compose preflight wrapper" >&2 + return 1 + } + if grep -Eq 'connector-secrets\.workspace-registry|docker compose' "$manual"; then + echo "$profile manual documents a bypassed Compose or copied connector override path" >&2 + return 1 + fi + echo "$profile manual requires generated connector override and Compose preflight passed" +} + compose_fixture() { local name="$1" directory="$2"; shift 2 ( @@ -130,6 +150,21 @@ verify_connector_fixture() { echo "core process sees connector bindings and secret files passed" } +verify_documented_operator_path() { + local profile="$1" directory="$2" connector_override + connector_override="$directory/connector-secrets.local.yaml" + "$root/scripts/generate-connector-secrets-override.sh" \ + --bindings-env "$directory/workspace-bindings.env" --operator-env "$directory/.env" \ + --output "$connector_override" >/dev/null + ( + cd "$directory" + "$root/scripts/compose-with-preflight.sh" --env-file .env \ + -f compose.workspace-registry.yaml -f git-ssh.workspace-registry.yaml \ + -f connector-secrets.local.yaml config --quiet + ) + echo "$profile documented generator and preflight fixture passed" +} + verify_copied_operator_fixtures() { local fixture_root local_dir server_dir https_dir ssh_dir connector_dir fixture_root="$(mktemp -d "${TMPDIR%/}/thoth-install-fixtures.XXXXXX")" @@ -177,6 +212,22 @@ verify_copied_operator_fixtures() { prepare_binding_fixture "$ssh_dir" compose_fixture "copied SSH Git override fixture" "$ssh_dir" -f compose.workspace-registry.yaml -f git-ssh.yaml + : >"$server_dir/git-ssh-key"; : >"$server_dir/git-known-hosts" + : >"$server_dir/dwh-password"; : >"$server_dir/vector-api-key" + printf '%s\n' \ + "THT_WORKSPACE_GIT_SSH_KEY_FILE=$server_dir/git-ssh-key" \ + "THT_WORKSPACE_GIT_KNOWN_HOSTS_FILE=$server_dir/git-known-hosts" \ + "THT_WS_NORTH_STAR_RESEARCH_DWH_PASSWORD_SOURCE=$server_dir/dwh-password" \ + "THT_WS_NORTH_STAR_RESEARCH_VECTOR_API_KEY_SOURCE=$server_dir/vector-api-key" >>"$server_dir/.env" + cp "$root/docs/install/examples/git-ssh.workspace-registry.yaml" "$server_dir/git-ssh.workspace-registry.yaml" + : >"$ssh_dir/dwh-password"; : >"$ssh_dir/vector-api-key" + printf '%s\n' \ + "THT_WS_NORTH_STAR_RESEARCH_DWH_PASSWORD_SOURCE=$ssh_dir/dwh-password" \ + "THT_WS_NORTH_STAR_RESEARCH_VECTOR_API_KEY_SOURCE=$ssh_dir/vector-api-key" >>"$ssh_dir/.env" + cp "$root/docs/install/examples/git-ssh.workspace-registry.yaml" "$ssh_dir/git-ssh.workspace-registry.yaml" + verify_documented_operator_path local "$ssh_dir" + verify_documented_operator_path server "$server_dir" + cp "$root/docs/install/examples/local-compose.workspace-registry.yaml" "$connector_dir/compose.workspace-registry.yaml" : >"$connector_dir/dwh-password"; : >"$connector_dir/vector-password" printf '%s\n' \ @@ -218,6 +269,8 @@ verify_copied_operator_fixtures() { case "$profile" in --fixtures-only) [[ $# -eq 1 ]] || { echo "usage: $0 --fixtures-only" >&2; exit 2; } + verify_manual_supported_path local "$root/docs/install/local-workspace-registry.md" + verify_manual_supported_path server "$root/docs/install/server-workspace-registry.md" verify_copied_operator_fixtures exit 0 ;; @@ -292,28 +345,11 @@ verify_path_variable_values "$example" verify_path_variable_values "$root/docs/install/examples/git-https.workspace-registry.yaml" verify_path_variable_values "$root/docs/install/examples/git-ssh.workspace-registry.yaml" verify_path_variable_values "$root/docs/install/examples/workspace-bindings.env.example" -verify_path_variable_values "$root/docs/install/examples/connector-secrets.workspace-registry.yaml" verify_server_public_contract +verify_manual_supported_path "$profile" "$manual" -commands="$(mktemp "${TMPDIR:-/tmp}/thoth-install-docs.XXXXXX")" -trap 'rm -f "$commands"' EXIT HUP INT TERM - -# A runnable documentation command is a sh fence immediately following this marker. Commands -# outside the marker are explanatory/operator commands and are deliberately never executed here. -awk ' - /^[[:space:]]*$/ { marked=1; next } - marked && /^```(sh|bash|shell)[[:space:]]*$/ { in_fence=1; marked=0; seen=1; next } - in_fence && /^```[[:space:]]*$/ { in_fence=0; next } - in_fence { print } -' "$manual" >"$commands" - -[[ -s "$commands" ]] || { echo "no marked runnable commands in $profile manual" >&2; exit 1; } - -echo "== Validate $profile documented Compose example ==" -( - cd "$root" - bash "$commands" -) +echo "== Validate copied operator fixtures and documented optional Git transports ==" +verify_copied_operator_fixtures echo "== Run isolated workspace-registry bootstrap and recovery smoke ==" ( @@ -321,7 +357,4 @@ echo "== Run isolated workspace-registry bootstrap and recovery smoke ==" env -u WORKSPACE_GIT_REMOTE ./scripts/workspace-registry-smoke.sh ) -echo "== Validate copied operator fixtures and optional Git transports ==" -verify_copied_operator_fixtures - echo "$profile installation documentation verification passed"