From 53d257fb76969fa88377c7cfd67fcdd1654d3a76 Mon Sep 17 00:00:00 2001 From: mptyl Date: Wed, 5 Aug 2026 08:35:13 +0200 Subject: [PATCH] docs: add autonomous local installation guide --- .../examples/thothii-installation.local.yaml | 8 + docs/install/local-workspace-registry.md | 7 + docs/install/local.md | 306 ++++++++++++++++++ docs/install/pi-management.md | 143 ++++++++ docs/install/windows-line-endings.md | 80 +++++ scripts/test-verify-workspace-install-docs.sh | 4 + scripts/verify-workspace-install-docs.sh | 231 +++++++++++++ 7 files changed, 779 insertions(+) create mode 100644 docs/install/examples/thothii-installation.local.yaml create mode 100644 docs/install/local.md create mode 100644 docs/install/pi-management.md create mode 100644 docs/install/windows-line-endings.md diff --git a/docs/install/examples/thothii-installation.local.yaml b/docs/install/examples/thothii-installation.local.yaml new file mode 100644 index 00000000..60715e5d --- /dev/null +++ b/docs/install/examples/thothii-installation.local.yaml @@ -0,0 +1,8 @@ +# Copy this file to an operator-controlled path named exactly thothii-installation.yaml. +# Replace every absolute placeholder. Select exactly one Git transport override. +profile: local +projectDirectory: "/absolute/path/to/ThothII" +envFile: "/absolute/path/to/ThothII/deploy/env/local.env" +overrides: + - "/absolute/path/to/ThothII/deploy/compose.git-ssh.yaml" + - "/absolute/path/to/thothii-operator/connector-secrets.local.yaml" diff --git a/docs/install/local-workspace-registry.md b/docs/install/local-workspace-registry.md index 1e06a900..d77b38c1 100644 --- a/docs/install/local-workspace-registry.md +++ b/docs/install/local-workspace-registry.md @@ -1,5 +1,9 @@ # Local workspace-registry installation (Mac and PC) +Complete the [local PC/Mac/Linux installation](local.md) first. This guide continues with the +Git-backed workspace source of truth, installation-local connector bindings, and diagnostics. Use +the [Pi management manual](pi-management.md) for provider configuration and image recovery. + This guide runs a single-user ThothII registry on Docker Desktop (macOS or Windows) or a local Linux Docker Engine. It is intentionally loopback-only. Git is shared; the checkout, connector bindings, credentials, and session data are local. Never put credentials in workspace YAML, Git, @@ -150,6 +154,9 @@ Select exactly one repository Git transport override, `deploy/compose.git-ssh.ya `deploy/compose.git-https.yaml`. A Compose env file is not a shell environment, so export only the non-secret paths required by the maintenance commands. Generate the connector override and render through the preflight wrapper, which rejects unsafe paths and combined SSH+HTTPS selection. +Record the selected Git and generated connector overrides in the operator +[`thothii-installation.yaml` example](examples/thothii-installation.local.yaml), using absolute +paths, so `thothctl` remains the ordinary lifecycle interface. ```sh export THT_SOURCE_ROOT=/absolute/path/to/ThothII diff --git a/docs/install/local.md b/docs/install/local.md new file mode 100644 index 00000000..f81751d6 --- /dev/null +++ b/docs/install/local.md @@ -0,0 +1,306 @@ +# Install ThothII on a local PC or Mac + +This guide installs one loopback-only ThothII on the same Windows, macOS, or Linux computer that +runs Docker. The supported application is one Docker Compose distribution containing exactly +`frontend` and `core`; Pi is pinned inside `core`. DWH, vector database, embedding, and LLM remain +external configurable services even when they run on this computer. + +No host Pi, Node.js, Python, Go toolchain, Docker socket in core, or browser shell is required. +Commands that contain example paths must be changed to absolute paths on your computer. + +## Choose your platform + +- **macOS:** use Terminal and Docker Desktop. Apple Silicon and Intel are supported by the local + image build. +- **Windows PowerShell:** use Docker Desktop with its WSL2 engine, Git for Windows, the Windows + build launcher, and `thothctl-windows-amd64.exe`. +- **Windows WSL2 (recommended):** enable Docker Desktop integration for your Linux distribution, + clone under `/home/` rather than `/mnt/c`, and follow the Linux shell commands. +- **Linux PC:** use Docker Engine plus the Compose v2 plugin and the Linux `thothctl` binary. + +Windows users must also read [Windows and WSL2 line endings](windows-line-endings.md) before the +first build. + +## Prerequisites + +Install only: + +1. Git 2.39 or newer. +2. Docker Desktop on macOS/Windows, or Docker Engine on Linux. +3. Docker Compose v2 (`docker compose`, not legacy `docker-compose`). +4. About 10 GB of free disk for source, images, build cache, and initial volumes. +5. Network access to the workspace Git remote and configured DWH/vector/embedding/LLM endpoints. + +Verify the tools: + +```sh +git --version +docker version +docker compose version +docker run --rm hello-world +``` + +On Linux, add the operator to the Docker group only if local policy permits it; sign out and back +in afterward. A local installation needs no inbound firewall rule because ports bind only to +`127.0.0.1`. + +## Clone and verify LF + +Use a `git clone` command that disables automatic CRLF conversion for this checkout. + +macOS and Linux: + +```sh +git -c core.autocrlf=false clone https://github.example.invalid/your-org/ThothII.git +cd ThothII +git config --local core.autocrlf false +bash scripts/verify-line-endings.sh +``` + +Windows PowerShell: + +```powershell +git -c core.autocrlf=false clone https://github.example.invalid/your-org/ThothII.git +Set-Location ThothII +git config --local core.autocrlf false +& "C:\Program Files\Git\bin\bash.exe" scripts/verify-line-endings.sh +``` + +Windows WSL2: + +```sh +mkdir -p "$HOME/src" && cd "$HOME/src" +git -c core.autocrlf=false clone https://github.example.invalid/your-org/ThothII.git +cd ThothII +git config --local core.autocrlf false +bash scripts/verify-line-endings.sh +``` + +Stop if the verifier names any path. Do not build from a CRLF checkout. + +## Create the local operator files + +Copy the non-secret template. This untracked `.env` contains addresses and absolute source paths, +never secret values: + +```sh +cp deploy/env/local.env.example deploy/env/local.env +mkdir -p /absolute/path/to/thothii-operator/secrets +chmod 0700 /absolute/path/to/thothii-operator/secrets +``` + +Edit `deploy/env/local.env`. At minimum set the workspace Git remote, `PI_AUTH_FILE`, +`THT_SECRETS_FILE`, external service endpoints, and the absolute +`THT_WORKSPACE_BINDINGS_ENV_FILE`. Create each secret as a separate regular file under the +protected operator directory and set mode `0600`. On Windows use a user-only ACL instead. + +Do not paste credentials into this guide's commands, `.env`, workspace YAML, Git, URLs, image build +arguments, or the installation descriptor. Secret contents are mounted read-only under +`/run/secrets` (Pi's auth store has its own protected read-only mount) and must never be committed, +embedded, rendered, or logged. + +Follow [the local workspace-registry guide](local-workspace-registry.md) to create the bindings +file, choose exactly one Git SSH/HTTPS override, and generate the connector-secret override. A +fresh install requires a valid private workspace repository; the Git-backed registry remains the +source of truth. + +Copy the installation example to an operator-controlled file named exactly +`thothii-installation.yaml`, then replace all placeholders with absolute paths: + +```sh +cp docs/install/examples/thothii-installation.local.yaml \ + /absolute/path/to/thothii-operator/thothii-installation.yaml +``` + +For HTTPS, replace the SSH override in that file with `deploy/compose.git-https.yaml`. Add only +reviewed local overrides, including the generated connector-secret file. Paths may contain spaces +when correctly represented as YAML strings. + +Native Windows uses the same four fields. Use single-quoted absolute Windows paths so backslashes +remain literal YAML characters: + +```yaml +profile: local +projectDirectory: 'C:\Users\operator\src\ThothII' +envFile: 'C:\Users\operator\src\ThothII\deploy\env\local.env' +overrides: + - 'C:\Users\operator\src\ThothII\deploy\compose.git-ssh.yaml' + - 'C:\Users\operator\thothii-operator\connector-secrets.local.yaml' +``` + +## Address external services + +An address is interpreted inside `core`. Therefore container 127.0.0.1 means the container itself, +not the Docker host. Keep every DWH, vector, embedding, and LLM address configurable in the local +environment/workspace bindings. + +- **Docker Desktop (macOS and Windows):** use `host.docker.internal`, for example + `http://host.docker.internal:11434`. +- **Linux:** if a service runs on the host, create an untracked override and include its absolute + path in `thothii-installation.yaml`: + +```yaml +services: + core: + extra_hosts: + - "host.docker.internal:host-gateway" +``` + +Then use `host.docker.internal` in the endpoint. `extra_hosts: host.docker.internal:host-gateway` +is a host routing aid, not a bundled service. Prefer a real DNS name for independently operated +services; retain TLS and authentication even when co-located. + +## Build ThothII and thothctl + +From the repository root, macOS/Linux/WSL2 users run: + +```sh +bash scripts/build-local.sh +bash scripts/build-thothctl.sh +``` + +Native PowerShell users run: + +```powershell +powershell -ExecutionPolicy Bypass -File scripts/build-local.ps1 +& "C:\Program Files\Git\bin\bash.exe" scripts/build-thothctl.sh +``` + +The second command uses Docker to create native operator binaries under `dist/thothctl`; users do +not need to know or install Go. Select `thothctl-darwin-arm64` or `-amd64` on macOS, +`thothctl-linux-amd64` or `-arm64` on Linux/WSL2, and `thothctl-windows-amd64.exe` on Windows. +Copy the selected file to the protected operator directory and, on macOS/Linux, run `chmod 0755` +on it. + +## Start and verify + +Set convenient variables (PowerShell users use `$THTCTL` and `$INSTALLATION` with `& $THTCTL`): +Every operator call has the form `thothctl --installation `. + +```sh +THTCTL=/absolute/path/to/thothii-operator/thothctl +INSTALLATION=/absolute/path/to/thothii-operator/thothii-installation.yaml +"$THTCTL" --installation "$INSTALLATION" update --check-only +"$THTCTL" --installation "$INSTALLATION" start +"$THTCTL" --installation "$INSTALLATION" status +"$THTCTL" --installation "$INSTALLATION" doctor +``` + +Native PowerShell uses the same order: + +```powershell +$THTCTL = 'C:\Users\operator\thothii-operator\thothctl.exe' +$INSTALLATION = 'C:\Users\operator\thothii-operator\thothii-installation.yaml' +& $THTCTL --installation $INSTALLATION update --check-only +& $THTCTL --installation $INSTALLATION start +& $THTCTL --installation $INSTALLATION status +& $THTCTL --installation $INSTALLATION doctor +``` + +Wait for both services, then check the same-origin frontend and direct loopback core: + +```sh +curl --fail http://127.0.0.1:8080/health +curl --fail http://127.0.0.1:8787/health +"$THTCTL" --installation "$INSTALLATION" pi doctor +"$THTCTL" --installation "$INSTALLATION" pi test +``` + +Open . If a check fails, run `thothctl ... logs` or `pi logs`; these are +bounded and sanitize declared secrets. Do not publish either loopback port. + +## Update an installation + +Commit or back up local operator changes first. Application source updates are separate from Pi +lifecycle updates: + +```sh +"$THTCTL" --installation "$INSTALLATION" stop +git status --short +git pull --ff-only +git config --local core.autocrlf false +bash scripts/verify-line-endings.sh +bash scripts/build-local.sh +bash scripts/build-thothctl.sh +"$THTCTL" --installation "$INSTALLATION" update --check-only +"$THTCTL" --installation "$INSTALLATION" start +curl --fail http://127.0.0.1:8080/health +``` + +On native Windows use the PowerShell build launcher and the Git-for-Windows Bash LF check. Review +release notes before updating. Pi has its own transactional `pi update` and rollback workflow in +[Pi management](pi-management.md); never install a package in the running container. + +## Back up and restore + +Back up before source/Pi updates and test restoration periodically. First stop cleanly: + +```sh +"$THTCTL" --installation "$INSTALLATION" stop +docker volume ls --format '{{.Name}}' | grep '^thothii-' +``` + +Identify the four exact volumes belonging to this installation: `settings`, `pi-state`, +`workspace-registry`, and `sessions`. Confirm their Compose project label with `docker volume +inspect`. For each exact volume, archive it to a protected backup directory: + +```sh +BACKUP_DIR=/absolute/path/to/backups/2026-08-05 +VOLUME=exact-installation-volume-name +mkdir -p "$BACKUP_DIR" +docker run --rm -v "$VOLUME:/source:ro" -v "$BACKUP_DIR:/backup" \ + alpine:3.22 tar -C /source -czf "/backup/$VOLUME.tgz" . +``` + +Native PowerShell can run the same read-only archive container: + +```powershell +$BackupDir = 'C:\Users\operator\thothii-backups\2026-08-05' +$Volume = 'exact-installation-volume-name' +New-Item -ItemType Directory -Force $BackupDir | Out-Null +docker run --rm -v "${Volume}:/source:ro" -v "${BackupDir}:/backup" ` + alpine:3.22 tar -C /source -czf "/backup/${Volume}.tgz" . +``` + +Also back up the installation descriptor, operator environment, generated overrides, and secret +files to separate encrypted/protected storage. Never commit them. Record image digests and the Git +revision. Do not back up while containers are running. + +Restore only while stopped and only into a new, verified-empty exact target volume. Test the +archive in a disposable installation first: + +```sh +TARGET_VOLUME=exact-empty-target-volume-name +ARCHIVE=/absolute/path/to/backups/2026-08-05/exact-volume-name.tgz +docker run --rm -v "$TARGET_VOLUME:/target" alpine:3.22 \ + sh -c 'test -z "$(ls -A /target)"' +docker run --rm -v "$TARGET_VOLUME:/target" -v "$(dirname "$ARCHIVE"):/backup:ro" \ + alpine:3.22 tar -C /target -xzf "/backup/$(basename "$ARCHIVE")" +``` + +Restore all four volumes from the same backup set, restore protected operator files separately, +then run `update --check-only`, `start`, `doctor`, registry status/diagnostics, and a known session +before normal use. Never merge an archive into a non-empty volume. + +## Data-preserving uninstall + +Run `thothctl stop`, retain the installation descriptor at the same absolute path, and make one +verified backup set. In Docker Desktop, remove only this installation's stopped `core` and +`frontend` containers and optional local images; leave its four named volumes. On Linux, use the +containers' exact Compose project labels to remove only those stopped containers. Do not prune +global Docker data. + +Do **not** run `docker compose down --volumes`: it deletes the application data this procedure is +meant to preserve. Keep the operator directory and protected secrets if you intend to reinstall. +Using the same descriptor path preserves the `thothctl` project identity and reconnects the same +named volumes after rebuilding the source checkout. + +## Next: workspaces and Pi + +Complete [local workspace-registry installation](local-workspace-registry.md), including Git trust, +bindings, pull, validation, diagnostics, and registry recovery. Then use [Pi management](pi-management.md) +for provider/model configuration, smoke testing, transactional update, and rollback. + +The Git-backed workspace registry is always the workspace source of truth. Local DWH, vector, +embedding, or LLM processes remain independent services and are never added to the mandatory +ThothII core. diff --git a/docs/install/pi-management.md b/docs/install/pi-management.md new file mode 100644 index 00000000..3b6ee28d --- /dev/null +++ b/docs/install/pi-management.md @@ -0,0 +1,143 @@ +# Pi management + +Pi is pinned inside the ThothII `core` image. A local Pi, Node.js, Python, or Go installation is +not required. The browser can manage safe runtime settings, while the host-side `thothctl` +operator CLI performs container lifecycle and image updates. The core does not mount the Docker socket, +and there is no browser shell. + +In the commands below, replace `/absolute/path/to/thothii-installation.yaml` with the protected +installation descriptor created by the [local installation guide](local.md). + +## Who can use Pi Management + +On the loopback-only local profile (`AUTH_MODE=none`), the person using that PC can open Pi +Management and change installation defaults or run diagnostics. Do not expose ports 8080 or 8787 +to another machine. + +On a public server, Pi Management requires upstream authentication and a trusted administrator +claim supplied by the authenticated reverse proxy. Without that claim the API returns +`403 pi_management_forbidden`; ordinary users cannot change installation-wide Pi settings. Image +updates are never available from the web page on either profile. + +## Use the Pi Management page + +Open ThothII, choose **Pi Management**, and check the bundled version and readiness. The page: + +- offers only supported provider, model, and reasoning choices; +- saves non-secret defaults; +- reports credentials only as present or missing; +- runs a bounded provider smoke test; and +- shows at most 200 sanitized log lines. + +It never displays or accepts a credential, runs an image update, or opens a terminal. Per-user +browser preferences remain separate from installation defaults. + +## Use thothctl + +Set a short shell variable for the platform-specific executable. Examples below use macOS/Linux: + +```sh +THTCTL=/absolute/path/to/thothctl +INSTALLATION=/absolute/path/to/thothii-installation.yaml +``` + +The complete Pi command set is: + +```sh +"$THTCTL" --installation "$INSTALLATION" pi status +"$THTCTL" --installation "$INSTALLATION" pi doctor +"$THTCTL" --installation "$INSTALLATION" pi test +"$THTCTL" --installation "$INSTALLATION" pi check +"$THTCTL" --installation "$INSTALLATION" pi configure +"$THTCTL" --installation "$INSTALLATION" pi configure --provider zai --model glm-5.2 --thinking medium +"$THTCTL" --installation "$INSTALLATION" pi logs +"$THTCTL" --installation "$INSTALLATION" pi maintenance status +``` + +- `pi status` reads the image-bundled version. +- `pi doctor` checks the version boundaries, core health, settings, and configured model. +- `pi test` runs the isolated Pi/core smoke; `pi check` is its alias. +- `pi configure` presents closed choices on a terminal. Non-interactive use requires all three + flags. Never pass a credential as an argument. +- `pi logs` returns a bounded, sanitized snapshot and deliberately has no follow mode. +- `pi maintenance status` reports whether new session admission is gated. + +Use `thothctl status`, `doctor`, `logs`, `start`, `stop`, and `update --check-only` for the wider +installation. Direct Compose lifecycle commands can bypass the durable image selector and are not +the normal operator interface. + +## Handle credentials and secrets + +Keep provider credentials in the protected host file named by `PI_AUTH_FILE`, or in the documented +model secret file/bundle. Compose mounts protected material read-only under `/run/secrets` or at +Pi's protected auth path. Apply mode `0600` on macOS/Linux or a user-only ACL on Windows. + +Never put secret text in the installation YAML, operator environment, workspace Git repository, +command arguments, browser, screenshots, tickets, rendered Compose, or logs. Pi configuration is +declarative: executable `!command` values are rejected. Use supported environment references or +the protected credential files. + +After rotating a credential, restart core through `thothctl stop` and `thothctl start`, then run +`pi doctor` and `pi test`. Do not print the file while troubleshooting. + +## Update and roll back Pi + +Finish or close active work first. A build update uses source already present in this checkout: + +```sh +"$THTCTL" --installation "$INSTALLATION" pi update \ + --version 0.81.0 --source build --yes --drain +``` + +A registry update must use an immutable digest, never a mutable tag: + +```sh +"$THTCTL" --installation "$INSTALLATION" pi update \ + --version 0.81.0 --source pull \ + --image registry.example.invalid/thothii-core@sha256:<64-lowercase-hex-digits> \ + --yes --drain +``` + +The operation gates new sessions, records non-secret recovery state, recreates only `core`, checks +the requested version, health, settings, smoke request, configuration, and persistence mounts, +then promotes the verified image. Frontend and named volumes are preserved. + +To restore the image recorded by the interrupted or latest update: + +```sh +"$THTCTL" --installation "$INSTALLATION" pi rollback --yes +``` + +## Recover a failed update + +Do not delete `.thothctl`, `update-state.json`, `current-image.yaml`, containers, or volumes. First +inspect the durable gate and sanitized logs: + +```sh +"$THTCTL" --installation "$INSTALLATION" pi maintenance status +"$THTCTL" --installation "$INSTALLATION" pi logs +"$THTCTL" --installation "$INSTALLATION" pi rollback --yes +``` + +If rollback reports a terminal or stale maintenance state, repair the reported Docker, disk, or +configuration problem, then run: + +```sh +"$THTCTL" --installation "$INSTALLATION" pi maintenance recover --yes +"$THTCTL" --installation "$INSTALLATION" pi doctor +"$THTCTL" --installation "$INSTALLATION" pi test +``` + +If recovery still fails, leave maintenance active and preserve the recovery file. Collect only +sanitized `pi logs`, `status`, and `doctor` output for support; do not ungate the installation by +editing state files. + +## Direct support access + +Advanced support may inspect the bundled executable directly with the installation's exact +validated Compose file set, for example `docker compose exec core pi --version`. This is read-only +diagnosis, not an update mechanism. Do not run package installers, alter Pi files inside the live +container, mount the Docker socket, expose a browser shell, or use a host Pi as a substitute. + +Prefer `thothctl pi status`, `pi doctor`, `pi test`, and `pi logs`, because they include the durable +image selector and redact declared secret values. Share only their sanitized output. diff --git a/docs/install/windows-line-endings.md b/docs/install/windows-line-endings.md new file mode 100644 index 00000000..f0230295 --- /dev/null +++ b/docs/install/windows-line-endings.md @@ -0,0 +1,80 @@ +# Windows and WSL2 line endings + +ThothII's containers execute shell scripts from the source checkout. Those files must stay LF, +even when the PC normally uses CRLF. The repository's `.gitattributes` is authoritative, but a +Windows Git setting or an old checkout can still leave incorrect bytes. Check line endings after +every clone and pull, before building an image. + +## Recommended WSL2 clone + +Use Docker Desktop with WSL2 integration. Clone inside the Linux filesystem, for example under +`/home//src`, rather than under `/mnt/c`. This avoids slow cross-filesystem builds, +permission surprises, and Windows tools rewriting files behind WSL. + +```sh +mkdir -p "$HOME/src" +cd "$HOME/src" +git -c core.autocrlf=false clone https://github.example.invalid/your-org/ThothII.git +cd ThothII +git config --local core.autocrlf false +bash scripts/verify-line-endings.sh +``` + +Keep Docker Desktop's integration enabled for that WSL distribution. Run the Linux build scripts +and the Linux `thothctl` binary from the same WSL shell. + +## Repository-local LF policy + +Set the option in this repository only. Do not change a company-wide or personal Git policy just +for ThothII. + +```sh +git config --local core.autocrlf false +git config --local --get core.autocrlf +``` + +The second command must print `false`. `.gitattributes` keeps shell, YAML, Dockerfile, JSON, +TypeScript, Python, and Markdown files at LF; PowerShell files remain CRLF. + +For a native PowerShell clone, disable conversion during the first checkout and then store the +repository-local setting: + +```powershell +git -c core.autocrlf=false clone https://github.example.invalid/your-org/ThothII.git +Set-Location ThothII +git config --local core.autocrlf false +& "C:\Program Files\Git\bin\bash.exe" scripts/verify-line-endings.sh +``` + +## Verify after clone or pull + +From WSL2, Git Bash, macOS, or Linux run: + +```sh +bash scripts/verify-line-endings.sh +``` + +Success exits with code 0 and prints no offending path. If it lists a file, do not build or start +ThothII. Correct the checkout first. Native PowerShell users can invoke the same script through +Git for Windows as shown above. + +## Recover an existing CRLF clone + +The safest recovery is to reclone into a new directory. First commit wanted work or copy it to a +backup outside both clones. Then clone with conversion disabled, run the verifier, and copy back +only reviewed changes. + +If a reviewed working tree must be repaired in place, make a backup or commit all wanted changes +before continuing. Then use Git's repository attributes to stage a renormalization: + +```sh +git status --short +git config --local core.autocrlf false +git add --renormalize . +git diff --cached --check +git diff --cached +bash scripts/verify-line-endings.sh +``` + +Review every staged change before committing. The procedure intentionally avoids destructive Git +resets; replacing the clone is easier to audit and much safer for uncommitted work. diff --git a/scripts/test-verify-workspace-install-docs.sh b/scripts/test-verify-workspace-install-docs.sh index 6e7ead0c..f0c75d9b 100755 --- a/scripts/test-verify-workspace-install-docs.sh +++ b/scripts/test-verify-workspace-install-docs.sh @@ -9,6 +9,10 @@ trap 'rm -f "$output"' EXIT HUP INT TERM "$root/scripts/verify-workspace-install-docs.sh" --fixtures-only >"$output" for fixture in \ + "local installation guide contract" \ + "Windows line-ending recovery guide contract" \ + "Pi management guide contract" \ + "local installation example rendered from path with spaces" \ "local manual canonical base+override references" \ "server manual canonical base+override references" \ "canonical local base+override fixture" \ diff --git a/scripts/verify-workspace-install-docs.sh b/scripts/verify-workspace-install-docs.sh index d9875b96..5e6dde68 100755 --- a/scripts/verify-workspace-install-docs.sh +++ b/scripts/verify-workspace-install-docs.sh @@ -43,6 +43,132 @@ verify_path_variable_values() { done <"$source" } +require_headings() { + local source="$1" label="$2" + shift 2 + local heading + for heading in "$@"; do + grep -Fqx "## $heading" "$source" || { + echo "missing required heading in $label: $heading" >&2 + return 1 + } + done +} + +require_text() { + local source="$1" label="$2" + shift 2 + local expected + for expected in "$@"; do + grep -Fq -- "$expected" "$source" || { + echo "$label lacks required instruction: $expected" >&2 + return 1 + } + done +} + +verify_local_guide() { + local guide="$root/docs/install/local.md" + [[ -f "$guide" ]] || { + echo "missing local installation guide: docs/install/local.md" >&2 + return 1 + } + require_headings "$guide" "local installation guide" \ + "Choose your platform" \ + "Prerequisites" \ + "Clone and verify LF" \ + "Create the local operator files" \ + "Address external services" \ + "Build ThothII and thothctl" \ + "Start and verify" \ + "Update an installation" \ + "Back up and restore" \ + "Data-preserving uninstall" \ + "Next: workspaces and Pi" + require_text "$guide" "local installation guide" \ + "git clone" \ + "bash scripts/verify-line-endings.sh" \ + "deploy/env/local.env" \ + "host.docker.internal" \ + "host-gateway" \ + "container 127.0.0.1" \ + "bash scripts/build-local.sh" \ + "scripts/build-local.ps1" \ + "bash scripts/build-thothctl.sh" \ + "thothctl --installation" \ + "curl --fail http://127.0.0.1:8080/health" \ + "http://127.0.0.1:8080" \ + "git pull --ff-only" \ + "docker compose down --volumes" + echo "local installation guide contract passed" +} + +verify_windows_line_endings_guide() { + local guide="$root/docs/install/windows-line-endings.md" + [[ -f "$guide" ]] || { + echo "missing Windows line-ending guide: docs/install/windows-line-endings.md" >&2 + return 1 + } + require_headings "$guide" "Windows line-ending guide" \ + "Recommended WSL2 clone" \ + "Repository-local LF policy" \ + "Verify after clone or pull" \ + "Recover an existing CRLF clone" + require_text "$guide" "Windows line-ending guide" \ + "git config --local core.autocrlf false" \ + "bash scripts/verify-line-endings.sh" \ + "git add --renormalize ." \ + "git diff --cached --check" \ + "reclone" + if grep -Fq 'git reset --hard' "$guide"; then + node - "$guide" <<'NODE' +const fs = require("fs"); +const lines = fs.readFileSync(process.argv[2], "utf8").split(/\n/); +for (let index = 0; index < lines.length; index += 1) { + if (!lines[index].includes("git reset --hard")) continue; + const warning = lines.slice(Math.max(0, index - 4), index).join(" ").toLowerCase(); + if (!warning.includes("warning") || !warning.includes("destructive") || + !warning.includes("backup") || !warning.includes("commit")) { + throw new Error("git reset --hard lacks an immediate destructive warning requiring backup/commit"); + } +} +NODE + fi + echo "Windows line-ending recovery guide contract passed" +} + +verify_pi_management_guide() { + local guide="$root/docs/install/pi-management.md" + [[ -f "$guide" ]] || { + echo "missing Pi management guide: docs/install/pi-management.md" >&2 + return 1 + } + require_headings "$guide" "Pi management guide" \ + "Who can use Pi Management" \ + "Use the Pi Management page" \ + "Use thothctl" \ + "Handle credentials and secrets" \ + "Update and roll back Pi" \ + "Recover a failed update" \ + "Direct support access" + require_text "$guide" "Pi management guide" \ + "pi status" \ + "pi doctor" \ + "pi test" \ + "pi check" \ + "pi configure" \ + "pi update" \ + "pi rollback --yes" \ + "pi maintenance status" \ + "pi maintenance recover --yes" \ + "pi logs" \ + "/run/secrets" \ + "docker compose exec core pi" \ + "no browser shell" \ + "does not mount the Docker socket" + echo "Pi management guide contract passed" +} + verify_manual() { local profile="$1" manual manual="$root/docs/install/$profile-workspace-registry.md" @@ -93,6 +219,101 @@ verify_manual() { echo "$profile manual canonical base+override references passed" } +verify_local_installation_example() { + local example="$root/docs/install/examples/thothii-installation.local.yaml" + [[ -f "$example" ]] || { + echo "missing local installation example: docs/install/examples/thothii-installation.local.yaml" >&2 + return 1 + } + + local fixture source_copy operator_dir copied_example connector_override env_file + fixture="$(mktemp -d "${TMPDIR%/}/thoth local install.XXXXXX")" + trap 'rm -rf "$fixture"' RETURN + [[ "$fixture" == *" "* ]] || { + echo "local installation fixture path does not contain spaces" >&2 + return 1 + } + source_copy="$fixture/ThothII source" + operator_dir="$fixture/operator files" + mkdir -p "$source_copy/deploy/pi" "$operator_dir" + cp "$root/compose.yaml" "$source_copy/compose.yaml" + cp "$root/deploy/compose.local.yaml" "$source_copy/deploy/compose.local.yaml" + cp "$root/deploy/compose.git-ssh.yaml" "$source_copy/deploy/compose.git-ssh.yaml" + cp "$root/deploy/pi/models.json" "$source_copy/deploy/pi/models.json" + cp "$root/deploy/pi/settings.json" "$source_copy/deploy/pi/settings.json" + + write_private "$operator_dir/pi-auth.json" '{"zai":{"type":"api_key","key":"fixture-local-pi-key"}}' + write_private "$operator_dir/thothii.secrets" 'THT_MODEL_API_KEY=fixture-local-model-key' + write_private "$operator_dir/git-ssh-key" 'fixture-local-ssh-key' + write_private "$operator_dir/git-known-hosts" 'fixture-local-known-hosts' + write_private "$operator_dir/dwh-password" 'fixture-local-dwh-password' + printf '%s\n' \ + 'THT_WS_NORTH_STAR_RESEARCH_DWH_TRANSPORT=postgres_direct' \ + 'THT_WS_NORTH_STAR_RESEARCH_DWH_PASSWORD_FILE=/run/secrets/north-star-research-dwh-password' \ + >"$operator_dir/workspace-bindings.env" + env_file="$source_copy/deploy/env/local.env" + mkdir -p "$source_copy/deploy/env" + printf '%s\n' \ + 'THT_WORKSPACE_GIT_REMOTE=ssh://git@git.example.invalid/platform/thoth-workspaces.git' \ + "PI_AUTH_FILE=$operator_dir/pi-auth.json" \ + "THT_SECRETS_FILE=$operator_dir/thothii.secrets" \ + "THT_WORKSPACE_BINDINGS_ENV_FILE=$operator_dir/workspace-bindings.env" \ + "THT_WORKSPACE_GIT_SSH_KEY_FILE=$operator_dir/git-ssh-key" \ + "THT_WORKSPACE_GIT_KNOWN_HOSTS_FILE=$operator_dir/git-known-hosts" \ + "THT_WS_NORTH_STAR_RESEARCH_DWH_PASSWORD_SOURCE=$operator_dir/dwh-password" \ + >"$env_file" + connector_override="$operator_dir/connector-secrets.local.yaml" + "$root/scripts/generate-connector-secrets-override.sh" \ + --bindings-env "$operator_dir/workspace-bindings.env" \ + --operator-env "$env_file" \ + --output "$connector_override" >/dev/null + + copied_example="$fixture/thothii-installation.yaml" + local contents + contents="$(<"$example")" + contents="${contents//\/absolute\/path\/to\/ThothII/$source_copy}" + contents="${contents//\/absolute\/path\/to\/thothii-operator/$operator_dir}" + printf '%s\n' "$contents" >"$copied_example" + + local profile project_directory descriptor_env value + local -a overrides files + profile="$(sed -n 's/^profile: \([^[:space:]]*\)$/\1/p' "$copied_example")" + project_directory="$(sed -n 's/^projectDirectory: "\(.*\)"$/\1/p' "$copied_example")" + descriptor_env="$(sed -n 's/^envFile: "\(.*\)"$/\1/p' "$copied_example")" + while IFS= read -r value; do overrides+=("$value"); done < <(sed -n 's/^ - "\(.*\)"$/\1/p' "$copied_example") + [[ "$profile" == local && "$project_directory" == "$source_copy" && "$descriptor_env" == "$env_file" ]] || { + echo "local installation example does not resolve its required fields" >&2 + return 1 + } + [[ "${#overrides[@]}" -eq 2 && "${overrides[1]}" == "$connector_override" ]] || { + echo "local installation example does not select the expected optional overrides" >&2 + return 1 + } + files=(-f "$project_directory/compose.yaml" -f "$project_directory/deploy/compose.$profile.yaml") + for value in "${overrides[@]}"; do files+=(-f "$value"); done + local rendered="$fixture/local-installation.json" + "$root/scripts/compose-with-preflight.sh" --env-file "$descriptor_env" \ + "${files[@]}" config --format json >"$rendered" + node - "$rendered" <<'NODE' +const fs = require("fs"); +const config = JSON.parse(fs.readFileSync(process.argv[2], "utf8")); +if (Object.keys(config.services).sort().join(",") !== "core,frontend") { + throw new Error("local installation example must render exactly core,frontend"); +} +const output = JSON.stringify(config); +for (const secret of [ + "fixture-local-pi-key", + "fixture-local-model-key", + "fixture-local-ssh-key", + "fixture-local-known-hosts", + "fixture-local-dwh-password", +]) { + if (output.includes(secret)) throw new Error("local installation rendering exposed a fixture secret"); +} +NODE + echo "local installation example rendered from path with spaces passed" +} + write_private() { local path="$1" value="$2" printf '%s\n' "$value" >"$path" @@ -229,6 +450,10 @@ NODE case "$mode" in --fixtures-only) [[ $# -eq 1 ]] || { echo "usage: $0 --fixtures-only" >&2; exit 2; } + verify_local_guide + verify_windows_line_endings_guide + verify_pi_management_guide + verify_local_installation_example verify_manual local verify_manual server verify_compose_fixtures @@ -237,6 +462,12 @@ case "$mode" in profile="${2:-}" [[ $# -eq 2 && "$profile" =~ ^(local|server)$ ]] \ || { echo "usage: $0 --profile {local|server}" >&2; exit 2; } + if [[ "$profile" == local ]]; then + verify_local_guide + verify_windows_line_endings_guide + verify_pi_management_guide + verify_local_installation_example + fi verify_manual "$profile" verify_compose_fixtures echo "== Run isolated workspace-registry bootstrap and recovery smoke =="