fix(docs): enforce compose preflight workflow

2026-08-04 15:33:04 +02:00
parent eb01979072
commit 4f26a71156
6 changed files with 96 additions and 69 deletions
@@ -1,16 +0,0 @@
-# Reviewed direct PostgreSQL/pgvector connector-secret override for workspace-bindings.env.example.
-# Source variables are absolute host paths. Add only matching entries for the selected transport;
-# targets must equal the corresponding THT_WS_*_FILE paths in the bindings env file.
-services:
-  core:
-    secrets:
-      - source: psd_clinical_dwh_password
-        target: psd-clinical-dwh-password
-      - source: psd_clinical_vector_password
-        target: psd-clinical-vector-password
-
-secrets:
-  psd_clinical_dwh_password:
-    file: ${THT_WS_PSD_CLINICAL_DWH_PASSWORD_SOURCE:?set THT_WS_PSD_CLINICAL_DWH_PASSWORD_SOURCE}
-  psd_clinical_vector_password:
-    file: ${THT_WS_PSD_CLINICAL_VECTOR_PASSWORD_SOURCE:?set THT_WS_PSD_CLINICAL_VECTOR_PASSWORD_SOURCE}
@@ -1,5 +1,5 @@
 # Copy to an untracked operator file. This file contains only non-secret THT_WS_* bindings.
-# Every *_FILE value is a container path supplied by a reviewed connector-secret override.
+# Every *_FILE value is a container path supplied by the generated local connector override.
 THT_WS_PSD_CLINICAL_DWH_TRANSPORT=postgres_direct
 THT_WS_PSD_CLINICAL_DWH_HOST=dwh.internal.example
 THT_WS_PSD_CLINICAL_DWH_PORT=5432
@@ -80,10 +80,10 @@ file; a reader credential is never repurposed for writing.
 ## Direct PostgreSQL, REST, and SSH tunnel bindings

 Set only fields for the selected transport in the dedicated bindings env file. Canonical YAML keeps
-database/schema/collection, distance, embedding model, and dimensions shared in Git. Copy and
-review [the connector-secret override](examples/connector-secrets.workspace-registry.yaml) for the
-selected transport: every `*_FILE=/run/secrets/<target>` binding needs one matching Docker secret
-target and one host-only `*_SOURCE` path in operator `.env`.
+database/schema/collection, distance, embedding model, and dimensions shared in Git. Every
+`*_FILE=/run/secrets/<target>` binding needs one matching host-only `*_SOURCE` path in operator
+`.env`. Generate the untracked connector override from those two files during bootstrap; do not
+copy or maintain a workspace-specific Compose override.

 ```dotenv
 # Direct PostgreSQL and pgvector
@@ -134,23 +134,26 @@ before creating sessions. Git pull/push over SSH remains fully supported and is

 ## Bootstrap, first pull, and diagnostics

-Copy [the local Compose example](examples/local-compose.workspace-registry.yaml) and exactly one
-selected [SSH Git override](examples/git-ssh.workspace-registry.yaml) or [HTTPS Git override](examples/git-https.workspace-registry.yaml)
-plus [the bindings env example](examples/workspace-bindings.env.example) and a reviewed
-[connector-secret override](examples/connector-secrets.workspace-registry.yaml) into an untracked
-operator directory. Set `THT_SOURCE_ROOT` and the absolute `THT_WORKSPACE_BINDINGS_ENV_FILE` in
-its `.env`; this keeps the copied Compose file buildable and confines `THT_WS_*` values to `core`.
-Create only the host secret files named by the selected Git/connector override, then render it.
+Copy [the local Compose example](examples/local-compose.workspace-registry.yaml), exactly one
+selected [SSH Git override](examples/git-ssh.workspace-registry.yaml) or [HTTPS Git override](examples/git-https.workspace-registry.yaml),
+and [the bindings env example](examples/workspace-bindings.env.example) into an untracked operator
+directory. Set `THT_SOURCE_ROOT` and the absolute `THT_WORKSPACE_BINDINGS_ENV_FILE` in its `.env`;
+this keeps the copied Compose file buildable and confines `THT_WS_*` values to `core`. Create the
+host secret files named by the selected Git transport and every declared connector `*_SOURCE`, then
+generate the connector override and render through the preflight wrapper. The wrapper is required:
+it rejects unsafe source paths and a combined SSH+HTTPS Git selection before Compose runs.

-<!-- verify:command -->
 ```sh
-THT_SOURCE_ROOT="$(pwd -P)" THT_WORKSPACE_BINDINGS_ENV_FILE="$(pwd -P)/docs/install/examples/workspace-bindings.env.example" docker compose -f docs/install/examples/local-compose.workspace-registry.yaml config --quiet
+"$THT_SOURCE_ROOT/scripts/generate-connector-secrets-override.sh" --bindings-env "$THT_WORKSPACE_BINDINGS_ENV_FILE" --operator-env .env --output connector-secrets.local.yaml
+"$THT_SOURCE_ROOT/scripts/compose-with-preflight.sh" --env-file .env \
+  -f compose.workspace-registry.yaml -f git-ssh.workspace-registry.yaml -f connector-secrets.local.yaml config --quiet
 ```

 From the operator directory:

 ```sh
-docker compose -f compose.workspace-registry.yaml -f git-ssh.workspace-registry.yaml -f connector-secrets.workspace-registry.yaml up --build -d
+"$THT_SOURCE_ROOT/scripts/compose-with-preflight.sh" --env-file .env \
+  -f compose.workspace-registry.yaml -f git-ssh.workspace-registry.yaml -f connector-secrets.local.yaml up --build -d
 curl --fail --silent http://127.0.0.1:8787/health
 curl --fail --silent http://127.0.0.1:8787/workspace-registry/status
 curl --fail --silent http://127.0.0.1:8787/workspaces
@@ -55,7 +55,7 @@ mounts neither transport; add exactly one [HTTPS override](examples/git-https.wo
 or [SSH override](examples/git-ssh.workspace-registry.yaml). Strict host-key checking stays enabled
 and Git stderr is not exposed by the API. Rotate by atomically replacing the secret file,
 restarting `core`, and performing pull/status; never put the material in an environment variable or
-`docker compose config` output.
+rendered Compose output.

 ## Shared Git values, local bindings, and secret files

@@ -79,9 +79,9 @@ Variable names derive from the immutable ID: `psd-clinical` becomes `PSD_CLINICA
 `THT_WS_PSD_CLINICAL_DWH_PASSWORD_FILE`. A declared vector writer uses the distinct
 `THT_WS_PSD_CLINICAL_VECTOR_WRITER_API_KEY_FILE`; a reader file is never a writer substitute.
 Copy [the bindings env example](examples/workspace-bindings.env.example) to the protected operator
-directory. Every path-valued `*_FILE` entry must be supplied by a reviewed
-[connector-secret override](examples/connector-secrets.workspace-registry.yaml) with a matching
-Docker secret target below `/run/secrets` and an absolute host-only `*_SOURCE` path.
+directory. Every path-valued `*_FILE` entry needs an absolute host-only `*_SOURCE` path. Generate
+the untracked connector override from those files during bootstrap; do not copy or maintain a
+workspace-specific Compose override.

 ## Direct PostgreSQL, REST, and SSH tunnel bindings

@@ -142,27 +142,25 @@ Copy [the server Compose example](examples/server-compose.workspace-registry.yam
 selected Git override to the protected operator directory. Set `THT_SOURCE_ROOT` to the absolute
 ThothII checkout; a copied file cannot use a relative build context. Copy
 `deploy/workspaces/server-sessions.yaml.example` into that operator directory, review it, then set
-the absolute `THT_SERVER_WORKSPACE_CONFIG` path. Copy the bindings env example and a reviewed
-connector-secret override, then set absolute `THT_WORKSPACE_BINDINGS_ENV_FILE` and connector
-`*_SOURCE` paths. The same `.env` must set
+the absolute `THT_SERVER_WORKSPACE_CONFIG` path. Copy the bindings env example, then set absolute
+`THT_WORKSPACE_BINDINGS_ENV_FILE` and connector `*_SOURCE` paths. The same `.env` must set
 `THT_SESSION_DB_HOST`, `THT_SESSION_DB_NAME`, `THT_SESSION_RUNTIME_USER`,
 `THT_SESSION_RUNTIME_PASSWORD_SOURCE`, and `THT_SESSION_CA_SOURCE`; the base Compose file wires
 `postgres`, `verify-full`, and the two Docker secret mount paths. This is the public server profile,
 not a filesystem-session fallback.

-<!-- verify:command -->
-```sh
-./scripts/verify-workspace-install-docs.sh --fixtures-only
-```
-
 Configure the portal proxy so the frontend and `/api` share one origin. It authenticates first and
 forwards the trusted identity expected by `AUTH_MODE=upstream`; it is the only public listener.
 From a trusted maintenance shell:

 ```sh
-docker compose -f compose.workspace-registry.yaml -f git-ssh.workspace-registry.yaml -f connector-secrets.workspace-registry.yaml up --build -d
-docker compose -f compose.workspace-registry.yaml exec -T core curl --fail --silent http://127.0.0.1:8787/health
-docker compose -f compose.workspace-registry.yaml exec -T core curl --fail --silent http://127.0.0.1:8787/workspace-registry/status
+"$THT_SOURCE_ROOT/scripts/generate-connector-secrets-override.sh" --bindings-env "$THT_WORKSPACE_BINDINGS_ENV_FILE" --operator-env .env --output connector-secrets.local.yaml
+"$THT_SOURCE_ROOT/scripts/compose-with-preflight.sh" --env-file .env \
+  -f compose.workspace-registry.yaml -f git-ssh.workspace-registry.yaml -f connector-secrets.local.yaml up --build -d
+"$THT_SOURCE_ROOT/scripts/compose-with-preflight.sh" --env-file .env \
+  -f compose.workspace-registry.yaml -f git-ssh.workspace-registry.yaml -f connector-secrets.local.yaml exec -T core curl --fail --silent http://127.0.0.1:8787/health
+"$THT_SOURCE_ROOT/scripts/compose-with-preflight.sh" --env-file .env \
+  -f compose.workspace-registry.yaml -f git-ssh.workspace-registry.yaml -f connector-secrets.local.yaml exec -T core curl --fail --silent http://127.0.0.1:8787/workspace-registry/status
 ```

 `/health` is liveness. Registry status verifies branch/head/degraded state and the active validated
@@ -9,6 +9,8 @@ trap 'rm -f "$output"' EXIT HUP INT TERM
 "$root/scripts/verify-workspace-install-docs.sh" --fixtures-only >"$output"

 for fixture in \
+  "local manual requires generated connector override and Compose preflight" \
+  "server manual requires generated connector override and Compose preflight" \
  "copied local base fixture" \
  "copied server PostgreSQL/TLS fixture" \
  "copied HTTPS Git override fixture" \
@@ -25,3 +27,10 @@ for fixture in \
    exit 1
  }
 done
+
+if rg -n 'connector-secrets\.workspace-registry|docker compose' \
+  "$root/docs/install/local-workspace-registry.md" \
+  "$root/docs/install/server-workspace-registry.md"; then
+  echo "installation manuals still document a bypassed Compose or copied connector override path" >&2
+  exit 1
+fi
@@ -60,6 +60,26 @@ verify_server_public_contract() {
  done
 }

+verify_manual_supported_path() {
+  local profile="$1" manual="$2"
+  local generator='"$THT_SOURCE_ROOT/scripts/generate-connector-secrets-override.sh" --bindings-env "$THT_WORKSPACE_BINDINGS_ENV_FILE" --operator-env .env --output connector-secrets.local.yaml'
+  local wrapper='"$THT_SOURCE_ROOT/scripts/compose-with-preflight.sh" --env-file .env'
+
+  grep -Fq "$generator" "$manual" || {
+    echo "$profile manual does not document the connector override generator" >&2
+    return 1
+  }
+  grep -Fq "$wrapper" "$manual" || {
+    echo "$profile manual does not document the Compose preflight wrapper" >&2
+    return 1
+  }
+  if grep -Eq 'connector-secrets\.workspace-registry|docker compose' "$manual"; then
+    echo "$profile manual documents a bypassed Compose or copied connector override path" >&2
+    return 1
+  fi
+  echo "$profile manual requires generated connector override and Compose preflight passed"
+}
+
 compose_fixture() {
  local name="$1" directory="$2"; shift 2
  (
@@ -130,6 +150,21 @@ verify_connector_fixture() {
  echo "core process sees connector bindings and secret files passed"
 }

+verify_documented_operator_path() {
+  local profile="$1" directory="$2" connector_override
+  connector_override="$directory/connector-secrets.local.yaml"
+  "$root/scripts/generate-connector-secrets-override.sh" \
+    --bindings-env "$directory/workspace-bindings.env" --operator-env "$directory/.env" \
+    --output "$connector_override" >/dev/null
+  (
+    cd "$directory"
+    "$root/scripts/compose-with-preflight.sh" --env-file .env \
+      -f compose.workspace-registry.yaml -f git-ssh.workspace-registry.yaml \
+      -f connector-secrets.local.yaml config --quiet
+  )
+  echo "$profile documented generator and preflight fixture passed"
+}
+
 verify_copied_operator_fixtures() {
  local fixture_root local_dir server_dir https_dir ssh_dir connector_dir
  fixture_root="$(mktemp -d "${TMPDIR%/}/thoth-install-fixtures.XXXXXX")"
@@ -177,6 +212,22 @@ verify_copied_operator_fixtures() {
  prepare_binding_fixture "$ssh_dir"
  compose_fixture "copied SSH Git override fixture" "$ssh_dir" -f compose.workspace-registry.yaml -f git-ssh.yaml

+  : >"$server_dir/git-ssh-key"; : >"$server_dir/git-known-hosts"
+  : >"$server_dir/dwh-password"; : >"$server_dir/vector-api-key"
+  printf '%s\n' \
+    "THT_WORKSPACE_GIT_SSH_KEY_FILE=$server_dir/git-ssh-key" \
+    "THT_WORKSPACE_GIT_KNOWN_HOSTS_FILE=$server_dir/git-known-hosts" \
+    "THT_WS_NORTH_STAR_RESEARCH_DWH_PASSWORD_SOURCE=$server_dir/dwh-password" \
+    "THT_WS_NORTH_STAR_RESEARCH_VECTOR_API_KEY_SOURCE=$server_dir/vector-api-key" >>"$server_dir/.env"
+  cp "$root/docs/install/examples/git-ssh.workspace-registry.yaml" "$server_dir/git-ssh.workspace-registry.yaml"
+  : >"$ssh_dir/dwh-password"; : >"$ssh_dir/vector-api-key"
+  printf '%s\n' \
+    "THT_WS_NORTH_STAR_RESEARCH_DWH_PASSWORD_SOURCE=$ssh_dir/dwh-password" \
+    "THT_WS_NORTH_STAR_RESEARCH_VECTOR_API_KEY_SOURCE=$ssh_dir/vector-api-key" >>"$ssh_dir/.env"
+  cp "$root/docs/install/examples/git-ssh.workspace-registry.yaml" "$ssh_dir/git-ssh.workspace-registry.yaml"
+  verify_documented_operator_path local "$ssh_dir"
+  verify_documented_operator_path server "$server_dir"
+
  cp "$root/docs/install/examples/local-compose.workspace-registry.yaml" "$connector_dir/compose.workspace-registry.yaml"
  : >"$connector_dir/dwh-password"; : >"$connector_dir/vector-password"
  printf '%s\n' \
@@ -218,6 +269,8 @@ verify_copied_operator_fixtures() {
 case "$profile" in
  --fixtures-only)
    [[ $# -eq 1 ]] || { echo "usage: $0 --fixtures-only" >&2; exit 2; }
+    verify_manual_supported_path local "$root/docs/install/local-workspace-registry.md"
+    verify_manual_supported_path server "$root/docs/install/server-workspace-registry.md"
    verify_copied_operator_fixtures
    exit 0
    ;;
@@ -292,28 +345,11 @@ verify_path_variable_values "$example"
 verify_path_variable_values "$root/docs/install/examples/git-https.workspace-registry.yaml"
 verify_path_variable_values "$root/docs/install/examples/git-ssh.workspace-registry.yaml"
 verify_path_variable_values "$root/docs/install/examples/workspace-bindings.env.example"
-verify_path_variable_values "$root/docs/install/examples/connector-secrets.workspace-registry.yaml"
 verify_server_public_contract
+verify_manual_supported_path "$profile" "$manual"

-commands="$(mktemp "${TMPDIR:-/tmp}/thoth-install-docs.XXXXXX")"
-trap 'rm -f "$commands"' EXIT HUP INT TERM
-
-# A runnable documentation command is a sh fence immediately following this marker.  Commands
-# outside the marker are explanatory/operator commands and are deliberately never executed here.
-awk '
-  /^<!--[[:space:]]*verify:command[[:space:]]*-->[[:space:]]*$/ { marked=1; next }
-  marked && /^```(sh|bash|shell)[[:space:]]*$/ { in_fence=1; marked=0; seen=1; next }
-  in_fence && /^```[[:space:]]*$/ { in_fence=0; next }
-  in_fence { print }
-' "$manual" >"$commands"
-
-[[ -s "$commands" ]] || { echo "no marked runnable commands in $profile manual" >&2; exit 1; }
-
-echo "== Validate $profile documented Compose example =="
-(
-  cd "$root"
-  bash "$commands"
-)
+echo "== Validate copied operator fixtures and documented optional Git transports =="
+verify_copied_operator_fixtures

 echo "== Run isolated workspace-registry bootstrap and recovery smoke =="
 (
@@ -321,7 +357,4 @@ echo "== Run isolated workspace-registry bootstrap and recovery smoke =="
  env -u WORKSPACE_GIT_REMOTE ./scripts/workspace-registry-smoke.sh
 )

-echo "== Validate copied operator fixtures and optional Git transports =="
-verify_copied_operator_fixtures
-
 echo "$profile installation documentation verification passed"