mptyl/ThothII
1
0
Fork 0
Code Issues 25 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

fix: bind workspace connector configuration safely

Browse Source
This commit is contained in:
marcopan
2026-08-04 08:22:09 +02:00
parent e5219deab1
commit 37404512ce
8 changed files with 137 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/install/examples/connector-secrets.workspace-registry.yaml
+16
View File
@@ -0,0 +1,16 @@
# Reviewed direct PostgreSQL/pgvector connector-secret override for workspace-bindings.env.example.
# Source variables are absolute host paths. Add only matching entries for the selected transport;
# targets must equal the corresponding THT_WS_*_FILE paths in the bindings env file.
services:
core:
secrets:
- source: psd_clinical_dwh_password
target: psd-clinical-dwh-password
- source: psd_clinical_vector_password
target: psd-clinical-vector-password
secrets:
psd_clinical_dwh_password:
file: ${THT_WS_PSD_CLINICAL_DWH_PASSWORD_SOURCE:?set THT_WS_PSD_CLINICAL_DWH_PASSWORD_SOURCE}
psd_clinical_vector_password:
file: ${THT_WS_PSD_CLINICAL_VECTOR_PASSWORD_SOURCE:?set THT_WS_PSD_CLINICAL_VECTOR_PASSWORD_SOURCE}
docs/install/examples/local-compose.workspace-registry.yaml
+3
View File
@@ -8,6 +8,9 @@ services:
build:
context: ${THT_SOURCE_ROOT:?set THT_SOURCE_ROOT to the absolute ThothII source checkout}
dockerfile: docker/core.Dockerfile
env_file:
- path: ${THT_WORKSPACE_BINDINGS_ENV_FILE:?set THT_WORKSPACE_BINDINGS_ENV_FILE to an absolute THT_WS bindings file}
required: true
environment:
HOST: 0.0.0.0
PORT: "8787"
docs/install/examples/server-compose.workspace-registry.yaml
+3
View File
@@ -8,6 +8,9 @@ services:
build:
context: ${THT_SOURCE_ROOT:?set THT_SOURCE_ROOT to the absolute ThothII source checkout}
dockerfile: docker/core.Dockerfile
env_file:
- path: ${THT_WORKSPACE_BINDINGS_ENV_FILE:?set THT_WORKSPACE_BINDINGS_ENV_FILE to an absolute THT_WS bindings file}
required: true
environment:
HOST: 0.0.0.0
PORT: "8787"
docs/install/examples/workspace-bindings.env.example
+13
View File
@@ -0,0 +1,13 @@
# Copy to an untracked operator file. This file contains only non-secret THT_WS_* bindings.
# Every *_FILE value is a container path supplied by a reviewed connector-secret override.
THT_WS_PSD_CLINICAL_DWH_TRANSPORT=postgres_direct
THT_WS_PSD_CLINICAL_DWH_HOST=dwh.internal.example
THT_WS_PSD_CLINICAL_DWH_PORT=5432
THT_WS_PSD_CLINICAL_DWH_USER=thoth_reader
THT_WS_PSD_CLINICAL_DWH_PASSWORD_FILE=/run/secrets/psd-clinical-dwh-password
THT_WS_PSD_CLINICAL_VECTOR_TRANSPORT=pgvector_direct
THT_WS_PSD_CLINICAL_VECTOR_HOST=vector.internal.example
THT_WS_PSD_CLINICAL_VECTOR_PORT=5432
THT_WS_PSD_CLINICAL_VECTOR_USER=thoth_vector_reader
THT_WS_PSD_CLINICAL_VECTOR_PASSWORD_FILE=/run/secrets/psd-clinical-vector-password
THT_WS_PSD_CLINICAL_EMBEDDING_BASE_URL=https://embeddings.internal.example
docs/install/local-workspace-registry.md
+18 -9
View File
@@ -55,7 +55,8 @@ and branch are non-secret; every `*_FILE` is a local path whose content never en
| Location | Contains | Never contains |
| --- | --- | --- |
| Git workspace repository | schema v2 YAML, generated binding names, LLM policy, model/index identity | installation hostnames, keys, passwords, certificates, SSH keys |
| local `.env` | remote, branch, installation ID, selected transport and endpoints | secret contents |
| local `.env` | remote, branch, installation ID, `THT_WORKSPACE_BINDINGS_ENV_FILE`, and secret source paths | secret contents or `THT_WS_*` values |
| workspace bindings env file | only `THT_WS_*` transport, endpoint, user, and `/run/secrets/...` path bindings | secret contents or unrelated application settings |
| local secret directory | Git credentials/key, known hosts, CA, connector secret files | a copied registry checkout |
| Docker volumes | registry checkout/snapshots/state/locks and local data | host-only secret source files |
@@ -69,14 +70,20 @@ locks/ # short-lived publish locks
```
Installation variables are deterministic: `psd-clinical` becomes `PSD_CLINICAL`, and every name
is `THT_WS_<NAMESPACE>_<ROLE>_<SUFFIX>`. Credentials and certificates use `*_FILE` path variables.
is `THT_WS_<NAMESPACE>_<ROLE>_<SUFFIX>`. Copy
[the bindings env example](examples/workspace-bindings.env.example) to an untracked operator file
and set its absolute path as `THT_WORKSPACE_BINDINGS_ENV_FILE`. It is loaded only into `core`.
Credentials and certificates use `*_FILE` path variables that must point inside `/run/secrets`.
If declared, `THT_WS_PSD_CLINICAL_VECTOR_WRITER_API_KEY_FILE` is distinct from the vector reader
file; a reader credential is never repurposed for writing.
## Direct PostgreSQL, REST, and SSH tunnel bindings
Set only fields for the selected transport. Canonical YAML keeps database/schema/collection,
distance, embedding model, and dimensions shared in Git.
Set only fields for the selected transport in the dedicated bindings env file. Canonical YAML keeps
database/schema/collection, distance, embedding model, and dimensions shared in Git. Copy and
review [the connector-secret override](examples/connector-secrets.workspace-registry.yaml) for the
selected transport: every `*_FILE=/run/secrets/<target>` binding needs one matching Docker secret
target and one host-only `*_SOURCE` path in operator `.env`.
```dotenv
# Direct PostgreSQL and pgvector
@@ -125,19 +132,21 @@ rather than weakening TLS; use runtime-trusted HTTPS or verified direct/SSH nati
Copy [the local Compose example](examples/local-compose.workspace-registry.yaml) and exactly one
selected [SSH Git override](examples/git-ssh.workspace-registry.yaml) or [HTTPS Git override](examples/git-https.workspace-registry.yaml)
into an untracked operator directory. Set `THT_SOURCE_ROOT` in its `.env` to the absolute source
checkout path; this keeps the copied Compose file buildable. Create only the secret files used by
the selected override, then render it before start.
plus [the bindings env example](examples/workspace-bindings.env.example) and a reviewed
[connector-secret override](examples/connector-secrets.workspace-registry.yaml) into an untracked
operator directory. Set `THT_SOURCE_ROOT` and the absolute `THT_WORKSPACE_BINDINGS_ENV_FILE` in
its `.env`; this keeps the copied Compose file buildable and confines `THT_WS_*` values to `core`.
Create only the host secret files named by the selected Git/connector override, then render it.
<!-- verify:command -->
```sh
THT_SOURCE_ROOT="$(pwd -P)" docker compose -f docs/install/examples/local-compose.workspace-registry.yaml config --quiet
THT_SOURCE_ROOT="$(pwd -P)" THT_WORKSPACE_BINDINGS_ENV_FILE="$(pwd -P)/docs/install/examples/workspace-bindings.env.example" docker compose -f docs/install/examples/local-compose.workspace-registry.yaml config --quiet
```
From the operator directory:
```sh
docker compose -f compose.workspace-registry.yaml -f git-ssh.workspace-registry.yaml up --build -d
docker compose -f compose.workspace-registry.yaml -f git-ssh.workspace-registry.yaml -f connector-secrets.workspace-registry.yaml up --build -d
curl --fail --silent http://127.0.0.1:8787/health
curl --fail --silent http://127.0.0.1:8787/workspace-registry/status
curl --fail --silent http://127.0.0.1:8787/workspaces
docs/install/server-workspace-registry.md
+12 -4
View File
@@ -61,8 +61,10 @@ restarting `core`, and performing pull/status; never put the material in an envi
Git describes workspace schema, immutable ID, DWH/vector identity, semantic-index dimensions and
distance, embedding contract, and LLM policy. The installation supplies remote/branch/installation
ID, transport, endpoints, users, ports, and `THT_WS_*` bindings. Secret contents are only in files,
never the values stored in Git or browser-local drafts.
ID and one absolute `THT_WORKSPACE_BINDINGS_ENV_FILE` containing only `THT_WS_*` transport,
endpoint, user, and `/run/secrets/...` path bindings. The base Compose loads that file only into
`core`. Secret contents are only in host files, never the values stored in Git or browser-local
drafts.
The runtime registry layout is persistent and must be backed up together:
@@ -76,6 +78,10 @@ The runtime registry layout is persistent and must be backed up together:
Variable names derive from the immutable ID: `psd-clinical` becomes `PSD_CLINICAL`, producing
`THT_WS_PSD_CLINICAL_DWH_PASSWORD_FILE`. A declared vector writer uses the distinct
`THT_WS_PSD_CLINICAL_VECTOR_WRITER_API_KEY_FILE`; a reader file is never a writer substitute.
Copy [the bindings env example](examples/workspace-bindings.env.example) to the protected operator
directory. Every path-valued `*_FILE` entry must be supplied by a reviewed
[connector-secret override](examples/connector-secrets.workspace-registry.yaml) with a matching
Docker secret target below `/run/secrets` and an absolute host-only `*_SOURCE` path.
## Direct PostgreSQL, REST, and SSH tunnel bindings
@@ -132,7 +138,9 @@ Copy [the server Compose example](examples/server-compose.workspace-registry.yam
selected Git override to the protected operator directory. Set `THT_SOURCE_ROOT` to the absolute
ThothII checkout; a copied file cannot use a relative build context. Copy
`deploy/workspaces/server-sessions.yaml.example` into that operator directory, review it, then set
the absolute `THT_SERVER_WORKSPACE_CONFIG` path. The same `.env` must set
the absolute `THT_SERVER_WORKSPACE_CONFIG` path. Copy the bindings env example and a reviewed
connector-secret override, then set absolute `THT_WORKSPACE_BINDINGS_ENV_FILE` and connector
`*_SOURCE` paths. The same `.env` must set
`THT_SESSION_DB_HOST`, `THT_SESSION_DB_NAME`, `THT_SESSION_RUNTIME_USER`,
`THT_SESSION_RUNTIME_PASSWORD_SOURCE`, and `THT_SESSION_CA_SOURCE`; the base Compose file wires
`postgres`, `verify-full`, and the two Docker secret mount paths. This is the public server profile,
@@ -148,7 +156,7 @@ forwards the trusted identity expected by `AUTH_MODE=upstream`; it is the only p
From a trusted maintenance shell:
```sh
docker compose -f compose.workspace-registry.yaml -f git-ssh.workspace-registry.yaml up --build -d
docker compose -f compose.workspace-registry.yaml -f git-ssh.workspace-registry.yaml -f connector-secrets.workspace-registry.yaml up --build -d
docker compose -f compose.workspace-registry.yaml exec -T core curl --fail --silent http://127.0.0.1:8787/health
docker compose -f compose.workspace-registry.yaml exec -T core curl --fail --silent http://127.0.0.1:8787/workspace-registry/status
```
scripts/test-verify-workspace-install-docs.sh
+2
View File
@@ -13,6 +13,8 @@ for fixture in \
"copied server PostgreSQL/TLS fixture" \
"copied HTTPS Git override fixture" \
"copied SSH Git override fixture" \
"copied connector binding/secret fixture" \
"core process sees connector bindings and secret files" \
"non-path secret-file fixture rejected"; do
grep -Fqx "$fixture passed" "$output" >/dev/null || {
echo "missing fixture verification: $fixture" >&2
scripts/verify-workspace-install-docs.sh
+70 -3
View File
@@ -59,16 +59,68 @@ compose_fixture() {
echo "$name passed"
}
prepare_binding_fixture() {
local directory="$1"
cp "$root/docs/install/examples/workspace-bindings.env.example" "$directory/workspace-bindings.env"
printf 'THT_WORKSPACE_BINDINGS_ENV_FILE=%s\n' "$directory/workspace-bindings.env" >>"$directory/.env"
}
verify_connector_fixture() {
local directory="$1" rendered project
project="thoth-install-connector-fixture-$$"
rendered="$(
cd "$directory"
docker compose --env-file .env -f compose.workspace-registry.yaml -f connector-secrets.yaml config
)"
for expected in \
'THT_WS_PSD_CLINICAL_DWH_TRANSPORT: postgres_direct' \
'THT_WS_PSD_CLINICAL_DWH_PASSWORD_FILE: /run/secrets/psd-clinical-dwh-password' \
'THT_WS_PSD_CLINICAL_VECTOR_PASSWORD_FILE: /run/secrets/psd-clinical-vector-password' \
'target: psd-clinical-dwh-password' \
'target: psd-clinical-vector-password'; do
grep -Fq "$expected" <<<"$rendered" || {
echo "connector fixture does not give core required binding or secret target: $expected" >&2
return 1
}
done
echo "copied connector binding/secret fixture passed"
if ! (
cd "$directory"
docker compose --project-name "$project" --env-file .env -f compose.workspace-registry.yaml -f connector-secrets.yaml \
run --rm --no-deps --build --entrypoint sh core -c '
test "$THT_WS_PSD_CLINICAL_DWH_TRANSPORT" = postgres_direct
test "$THT_WS_PSD_CLINICAL_DWH_PASSWORD_FILE" = /run/secrets/psd-clinical-dwh-password
test "$THT_WS_PSD_CLINICAL_VECTOR_PASSWORD_FILE" = /run/secrets/psd-clinical-vector-password
test -f "$THT_WS_PSD_CLINICAL_DWH_PASSWORD_FILE"
test -f "$THT_WS_PSD_CLINICAL_VECTOR_PASSWORD_FILE"
'
); then
(
cd "$directory"
docker compose --project-name "$project" --env-file .env -f compose.workspace-registry.yaml -f connector-secrets.yaml \
down --volumes --remove-orphans
) || true
return 1
fi
(
cd "$directory"
docker compose --project-name "$project" --env-file .env -f compose.workspace-registry.yaml -f connector-secrets.yaml \
down --volumes --remove-orphans
)
echo "core process sees connector bindings and secret files passed"
}
verify_copied_operator_fixtures() {
local fixture_root local_dir server_dir https_dir ssh_dir
local fixture_root local_dir server_dir https_dir ssh_dir connector_dir
fixture_root="$(mktemp -d "${TMPDIR:-/tmp}/thoth-install-fixtures.XXXXXX")"
trap 'rm -rf "$fixture_root"' RETURN
local_dir="$fixture_root/local"; server_dir="$fixture_root/server"
https_dir="$fixture_root/https"; ssh_dir="$fixture_root/ssh"
mkdir -p "$local_dir" "$server_dir" "$https_dir" "$ssh_dir"
https_dir="$fixture_root/https"; ssh_dir="$fixture_root/ssh"; connector_dir="$fixture_root/connector"
mkdir -p "$local_dir" "$server_dir" "$https_dir" "$ssh_dir" "$connector_dir"
cp "$root/docs/install/examples/local-compose.workspace-registry.yaml" "$local_dir/compose.workspace-registry.yaml"
printf 'THT_SOURCE_ROOT=%s\n' "$root" >"$local_dir/.env"
prepare_binding_fixture "$local_dir"
compose_fixture "copied local base fixture" "$local_dir" -f compose.workspace-registry.yaml
cp "$root/docs/install/examples/server-compose.workspace-registry.yaml" "$server_dir/compose.workspace-registry.yaml"
@@ -82,6 +134,7 @@ verify_copied_operator_fixtures() {
'THT_SESSION_RUNTIME_USER=thoth_sessions_app' \
"THT_SESSION_RUNTIME_PASSWORD_SOURCE=$server_dir/session-runtime-password" \
"THT_SESSION_CA_SOURCE=$server_dir/session-ca.pem" >"$server_dir/.env"
prepare_binding_fixture "$server_dir"
compose_fixture "copied server PostgreSQL/TLS fixture" "$server_dir" -f compose.workspace-registry.yaml
cp "$root/docs/install/examples/local-compose.workspace-registry.yaml" "$https_dir/compose.workspace-registry.yaml"
@@ -91,6 +144,7 @@ verify_copied_operator_fixtures() {
"THT_SOURCE_ROOT=$root" \
"THT_WORKSPACE_GIT_CREDENTIALS_FILE=$https_dir/git-credentials" \
"THT_WORKSPACE_GIT_CA_FILE=$https_dir/git-ca.pem" >"$https_dir/.env"
prepare_binding_fixture "$https_dir"
compose_fixture "copied HTTPS Git override fixture" "$https_dir" -f compose.workspace-registry.yaml -f git-https.yaml
cp "$root/docs/install/examples/local-compose.workspace-registry.yaml" "$ssh_dir/compose.workspace-registry.yaml"
@@ -100,8 +154,19 @@ verify_copied_operator_fixtures() {
"THT_SOURCE_ROOT=$root" \
"THT_WORKSPACE_GIT_SSH_KEY_FILE=$ssh_dir/git-ssh-key" \
"THT_WORKSPACE_GIT_KNOWN_HOSTS_FILE=$ssh_dir/git-known-hosts" >"$ssh_dir/.env"
prepare_binding_fixture "$ssh_dir"
compose_fixture "copied SSH Git override fixture" "$ssh_dir" -f compose.workspace-registry.yaml -f git-ssh.yaml
cp "$root/docs/install/examples/local-compose.workspace-registry.yaml" "$connector_dir/compose.workspace-registry.yaml"
cp "$root/docs/install/examples/connector-secrets.workspace-registry.yaml" "$connector_dir/connector-secrets.yaml"
: >"$connector_dir/dwh-password"; : >"$connector_dir/vector-password"
printf '%s\n' \
"THT_SOURCE_ROOT=$root" \
"THT_WS_PSD_CLINICAL_DWH_PASSWORD_SOURCE=$connector_dir/dwh-password" \
"THT_WS_PSD_CLINICAL_VECTOR_PASSWORD_SOURCE=$connector_dir/vector-password" >"$connector_dir/.env"
prepare_binding_fixture "$connector_dir"
verify_connector_fixture "$connector_dir"
printf 'THT_WS_EXAMPLE_DWH_PASSWORD_FILE=not-a-path\n' >"$fixture_root/non-path-secret.env"
if verify_secret_file_values "$fixture_root/non-path-secret.env" >/dev/null 2>&1; then
echo "non-path secret-file fixture was accepted" >&2
@@ -186,6 +251,8 @@ verify_secret_file_values "$manual"
verify_secret_file_values "$example"
verify_secret_file_values "$root/docs/install/examples/git-https.workspace-registry.yaml"
verify_secret_file_values "$root/docs/install/examples/git-ssh.workspace-registry.yaml"
verify_secret_file_values "$root/docs/install/examples/workspace-bindings.env.example"
verify_secret_file_values "$root/docs/install/examples/connector-secrets.workspace-registry.yaml"
verify_server_public_contract
commands="$(mktemp "${TMPDIR:-/tmp}/thoth-install-docs.XXXXXX")"