diff --git a/scripts/test-verify-workspace-install-docs.sh b/scripts/test-verify-workspace-install-docs.sh index 599450bb..d4be8964 100755 --- a/scripts/test-verify-workspace-install-docs.sh +++ b/scripts/test-verify-workspace-install-docs.sh @@ -1,1194 +1,57 @@ #!/usr/bin/env bash -# Regression test for copyable installation examples and secret-path validation. +# Regression tests for the current public install/workspace documentation verifier. set -euo pipefail root="$(cd "$(dirname "$0")/.." && pwd -P)" -output="$(mktemp "${TMPDIR:-/tmp}/thoth-install-docs-test.XXXXXX")" -verifier_functions="$(mktemp "${TMPDIR:-/tmp}/thoth-install-docs-functions.XXXXXX")" -negative_root="$(mktemp -d "${TMPDIR:-/tmp}/thoth-install-docs-negative.XXXXXX")" -trap 'rm -f "$output" "$verifier_functions"; rm -rf "$negative_root"' EXIT HUP INT TERM +fixture="$(mktemp -d "${TMPDIR:-/tmp}/thoth-install-docs-test.XXXXXX")" +trap 'rm -rf "$fixture"' EXIT HUP INT TERM -env -u TMPDIR -u THT_AUTH_CONFIG_ROOT \ - "$root/scripts/verify-workspace-install-docs.sh" --fixtures-only >"$output" - -env -u THT_AUTH_CONFIG_ROOT TMPDIR=/tmp/ \ - "$root/scripts/verify-workspace-install-docs.sh" --fixtures-only >"$output" - -for fixture in \ - "internal semantic infrastructure documentation contract" \ - "read-only workspace repository and encrypted runtime-secret contract" \ - "workspace Evidence documentation contract" \ - "local installation guide contract" \ - "source update fail-closed semantics" \ - "Windows line-ending recovery guide contract" \ - "Pi management guide contract" \ - "server installation guide contract" \ - "Nginx reverse-proxy guide contract" \ - "Caddy reverse-proxy guide contract" \ - "local installation example rendered from path with spaces" \ - "server installation example rendered from path with spaces" \ - "server pinned migration image fixture" \ - "server backup checksum root-only fixture" \ - "local manual canonical base+override references" \ - "server manual canonical base+override references" \ - "canonical local base+override fixture" \ - "canonical server base+override fixture" \ - "relative secret-source fixture rejected" \ - "CRLF recovery rewrites bytes and preserves mode-120000 symlinks"; do - grep -Fqx "$fixture passed" "$output" >/dev/null || { - echo "missing fixture verification: $fixture" >&2 - cat "$output" >&2 +expect_success() { + local marker="$1" + shift + local output + output="$($@)" + grep -Fqx -- "$marker" <<<"$output" || { + echo "missing verification marker: $marker" >&2 + printf '%s\n' "$output" >&2 exit 1 } -done +} -grep -Fq '## Internal Qdrant + Ollama semantic infrastructure' "$root/PROJECT_STATE.md" || { - echo "PROJECT_STATE.md does not record the internal Qdrant/Ollama snapshot" >&2 - exit 1 -} -grep -Fq 'Qdrant and Ollama are internal Compose services' "$root/AGENTS.md" || { - echo "AGENTS.md does not record the stable internal semantic-service guidance" >&2 - exit 1 -} -grep -Fq 'Do not add vector or embedding endpoint credentials to the bundle.' \ - "$root/deploy/secrets/README.md" || { - echo "secret bundle guide still permits vector/embedding runtime secrets" >&2 - exit 1 -} -legacy_pg_vector='engine: pg''vector' -legacy_ollama='provider: ollama''_compatible' -legacy_vector_binding='THT_WS__''VECTOR_TRANSPORT' -legacy_embedding_binding='THT_WS__''EMBEDDING_BASE_URL' -if rg -n "${legacy_pg_vector}|${legacy_ollama}|${legacy_vector_binding}|${legacy_embedding_binding}" \ - "$root/docs/workspace-diagnostic-protocol.md"; then - echo "workspace diagnostic protocol still documents external vector or embedding contracts" >&2 +expect_success 'Installation examples contract passed' \ + env -u TMPDIR -u THT_AUTH_CONFIG_ROOT "$root/scripts/verify-workspace-install-docs.sh" --fixtures-only +expect_success 'local installation documentation verification passed' \ + "$root/scripts/verify-workspace-install-docs.sh" --profile local +expect_success 'server installation documentation verification passed' \ + "$root/scripts/verify-workspace-install-docs.sh" --profile server + +if "$root/scripts/verify-workspace-install-docs.sh" --profile invalid >/dev/null 2>&1; then + echo 'invalid verification profile was accepted' >&2 exit 1 fi -server_guide="$root/docs/install/server.md" -grep -Fq 'scripts/prepare-server-pi-state.sh /srv/thothii/pi-state 10001 10001' "$server_guide" || { - echo "server guide does not initialize nested Pi-state targets before Compose" >&2 - exit 1 -} -grep -Fq 'sudo install -d -o "$operator_uid" -g 10001 -m 0750 /srv/thothii' "$server_guide" || { - echo "server operations guide does not set the parent traversal boundary" >&2 - exit 1 -} -for required in \ - 'does not require or permit creation' \ - 'getent passwd 10001' \ - 'getent group 10001' \ - 'chmod 0600 /srv/thothii/operator/server.env' \ - 'THT_BACKUP_ROOT=/srv/thothii-backups' \ - 'sessions migrate --yes' \ - '"pending":[]' \ - '"drifted":[]' \ - 'remove --yes' \ - 'sha256sum --check SHA256SUMS' \ - 'DOCKER-USER' \ - 'iptables -I INPUT' \ - 'com.docker.network.bridge.name'; do - grep -Fq -- "$required" "$server_guide" || { - echo "server operations guide lacks executable contract: $required" >&2 - exit 1 - } -done -for forbidden in 'sudo useradd' 'sudo groupadd' 'sudo usermod' 'sudo -u thothii' 'thothii-ops'; do - if grep -Fq -- "$forbidden" "$server_guide"; then - echo "server operations guide creates or depends on a host identity: $forbidden" >&2 - exit 1 - fi -done -grep -Fq '"$THT_BIN" --help' "$server_guide" || { - echo "server guide lacks plain tht --help" >&2 - exit 1 -} -if grep -Fq '"$THT_BIN" --installation "$INSTALLATION" --help' "$server_guide"; then - echo "server guide still uses installation-scoped --help" >&2 +mkdir -p "$fixture/docs" "$fixture/scripts" +cp -R "$root/docs/install" "$fixture/docs/install" +cp -R "$root/docs/operations" "$fixture/docs/operations" +cp "$root/docs/guida-utente.md" "$fixture/docs/guida-utente.md" +mkdir -p "$fixture/docs/architecture" "$fixture/docs/contracts" +cp "$root/docs/architecture/overview.md" "$fixture/docs/architecture/overview.md" +cp "$root/docs/contracts/catalog-schema-snapshot.md" "$fixture/docs/contracts/catalog-schema-snapshot.md" +cp "$root/mkdocs.yml" "$fixture/mkdocs.yml" +cp "$root/compose.yaml" "$fixture/compose.yaml" +cp "$root/scripts/run-stack.sh" "$fixture/scripts/run-stack.sh" + +# The fixture keeps the production navigation shape but lacks its canonical first-start page. +rm "$fixture/docs/install/first-start.md" +if THT_DOCS_VERIFY_ROOT="$fixture" "$root/scripts/verify-workspace-install-docs.sh" --fixtures-only \ + >"$fixture/missing-page.out" 2>&1; then + echo 'verifier accepted a missing first-start page' >&2 exit 1 fi - -for manual in "$root/docs/install/local-workspace-registry.md"; do - grep -Fq 'export THT_SOURCE_ROOT=/absolute/path/to/ThothII' "$manual" || { - echo "installation manual does not publish a self-contained THT_SOURCE_ROOT export: $manual" >&2 - exit 1 - } - if rg -n 'source[[:space:]]+\.env' "$manual"; then - echo "installation manual unsafely imports operator .env: $manual" >&2 - exit 1 - fi -done - -grep -Fq 'THT_BIN=/srv/thothii/operator/tht' \ - "$root/docs/install/server-workspace-registry.md" || { - echo "server installation manual does not use the installation-aware operator CLI" >&2 - exit 1 -} -grep -Fq 'INSTALLATION=/srv/thothii/operator/thothii-installation.yaml' \ - "$root/docs/install/server-workspace-registry.md" || { - echo "server installation manual does not identify the server installation descriptor" >&2 +grep -Fq 'missing required file: docs/install/first-start.md' "$fixture/missing-page.out" || { + echo 'missing first-start fixture failed for the wrong reason' >&2 + cat "$fixture/missing-page.out" >&2 exit 1 } -if rg -n 'local-compose\.workspace-registry|server-compose\.workspace-registry|connector-secrets\.workspace-registry|docker compose' \ - "$root/docs/install/local-workspace-registry.md" \ - "$root/docs/install/server-workspace-registry.md"; then - echo "installation manuals still document a bypassed Compose or copied connector override path" >&2 - exit 1 -fi - -# Load only the verifier's function definitions so each deliberately unsafe guide can be checked -# in isolation without invoking Docker-backed Compose fixtures. -sed '/^case "\$mode" in/,$d' "$root/scripts/verify-workspace-install-docs.sh" >"$verifier_functions" -# shellcheck source=/dev/null -source "$verifier_functions" - -prefix_tmpdir="$(TMPDIR=/tmp bash -c 'source "$1"; printf "%s" "$tmp_prefix"' _ "$verifier_functions")" -prefix_root="$(TMPDIR=/ bash -c 'source "$1"; printf "%s" "$tmp_prefix"' _ "$verifier_functions")" -[[ "$prefix_tmpdir" == "/tmp/" ]] || { - echo "normalized /tmp prefix must be slash-terminated: $prefix_tmpdir" >&2 - exit 1 -} -[[ "$prefix_root" == "/" ]] || { - echo "normalized root prefix must remain /: $prefix_root" >&2 - exit 1 -} -[[ "${prefix_root}thoth-probe.XXXXXX" == "/thoth-probe.XXXXXX" ]] || { - echo "root prefix concatenation produced an invalid path" >&2 - exit 1 -} - -PYTHONDONTWRITEBYTECODE=1 python3 "$root/scripts/test_workspace_descriptor_doc_contract.py" - -project_topology_fixture="$negative_root/project-topology-contradiction.md" -python3 - "$root/PROJECT_STATE.md" "$project_topology_fixture" <<'PY' -import pathlib, sys -source = pathlib.Path(sys.argv[1]).read_text() -marker = source.index("# Historical archive") -contradiction = ( - "The supported Compose stack is exactly `frontend` plus `core`.\n" - "DWH, vector DB, embedding, LLM, and reverse-proxy services are external endpoints.\n\n" -) -pathlib.Path(sys.argv[2]).write_text(source[:marker] + contradiction + source[marker:]) -PY -project_topology_output="$negative_root/project-topology-output" -set +e -verify_project_state_current_contract "$project_topology_fixture" contradictory-project-topology \ - >"$project_topology_output" 2>&1 -project_topology_status=$? -set -e -if [[ $project_topology_status -eq 0 ]] || \ - ! grep -Fq "contradictory active text" "$project_topology_output"; then - echo "contradictory current PROJECT_STATE topology was not rejected" >&2 - cat "$project_topology_output" >&2 - exit 1 -fi - -workspace_fixture="$negative_root/workspace-invalid.yaml" -python3 - "$root/deploy/workspaces/example.yaml" "$workspace_fixture" <<'PY' -import pathlib, sys, yaml -doc = yaml.safe_load(pathlib.Path(sys.argv[1]).read_text()) -doc["semantic_index"]["embedding"]["dimensions"] = 768 -pathlib.Path(sys.argv[2]).write_text(yaml.safe_dump(doc, sort_keys=False)) -PY -workspace_output="$negative_root/workspace-output" -set +e -verify_workspace_descriptor_semantic_contract "$workspace_fixture" invalid-workspace >"$workspace_output" 2>&1 -workspace_status=$? -set -e -if [[ $workspace_status -eq 0 ]] || ! grep -Fq "embedding dimensions must be 1024" "$workspace_output"; then - echo "semantic workspace fixture was not rejected correctly" >&2 - cat "$workspace_output" >&2 - exit 1 -fi - -local_manual_paraphrase="$negative_root/local-manual-paraphrase.md" -cp "$root/docs/install/local-workspace-registry.md" "$local_manual_paraphrase" -python3 - "$local_manual_paraphrase" <<'PY' -import pathlib, sys -path = pathlib.Path(sys.argv[1]) -text = path.read_text() -text = text.replace( - "| Workspace semantic index | Each workspace keeps exactly one Qdrant collection reserved for itself. | Schema, Evidence, and memory records share that one collection and are separated by the `kind` payload. |", - "| Workspace semantic index | A workspace keeps exactly one Qdrant collection reserved for itself. | Schema, Evidence, and Memory remain together in that collection and are still separated by payload `kind`. |", -) -path.write_text(text) -PY -semantic_index_spec="$(semantic_index_relationship_spec)" -verify_markdown_table_relationships "$local_manual_paraphrase" "local manual paraphrase" "Semantic index ownership contract" "$semantic_index_spec" >/dev/null - -production_paraphrase_root="$negative_root/production-paraphrase-root" -mkdir -p "$production_paraphrase_root" -rsync -a \ - --exclude '.git' \ - --exclude '.pytest_cache' \ - --exclude 'node_modules' \ - --exclude 'backend/node_modules' \ - --exclude 'frontend/node_modules' \ - --exclude 'harness/.venv' \ - "$root/" "$production_paraphrase_root/" -python3 - "$production_paraphrase_root/docs/install/local-workspace-registry.md" <<'PY' -import pathlib, sys -path = pathlib.Path(sys.argv[1]) -text = path.read_text() -text = text.replace( - "| Workspace semantic index | Each workspace keeps exactly one Qdrant collection reserved for itself. | Schema, Evidence, and memory records share that one collection and are separated by the `kind` payload. |", - "| Workspace semantic index | A workspace keeps exactly one Qdrant collection reserved for itself. | Schema, Evidence, and Memory remain together in that collection and are still separated by payload `kind`. |", -) -path.write_text(text) -PY -set +e -"$production_paraphrase_root/scripts/verify-workspace-install-docs.sh" --fixtures-only \ - >"$workspace_output" 2>&1 -workspace_status=$? -set -e -if [[ $workspace_status -ne 0 ]]; then - echo "production verifier rejected the accepted semantic-index paraphrase" >&2 - cat "$workspace_output" >&2 - exit 1 -fi - -local_manual_missing="$negative_root/local-manual-missing.md" -cp "$root/docs/install/local-workspace-registry.md" "$local_manual_missing" -python3 - "$local_manual_missing" <<'PY' -import pathlib, sys -path = pathlib.Path(sys.argv[1]) -text = path.read_text() -text = text.replace( - "| Workspace semantic index | Each workspace keeps exactly one Qdrant collection reserved for itself. | Schema, Evidence, and memory records share that one collection and are separated by the `kind` payload. |\n", - "", -) -path.write_text(text) -PY -set +e -verify_markdown_table_relationships "$local_manual_missing" "local manual missing ownership" "Semantic index ownership contract" "$semantic_index_spec" >"$workspace_output" 2>&1 -workspace_status=$? -set -e -if [[ $workspace_status -eq 0 ]]; then - echo "ownership omission fixture was not rejected correctly" >&2 - cat "$workspace_output" >&2 - exit 1 -fi - -local_manual_scattered="$negative_root/local-manual-scattered.md" -cp "$root/docs/install/local-workspace-registry.md" "$local_manual_scattered" -python3 - "$local_manual_scattered" <<'PY' -import pathlib, sys -path = pathlib.Path(sys.argv[1]) -text = path.read_text() -text = text.replace( - "| Workspace semantic index | Each workspace keeps exactly one Qdrant collection reserved for itself. | Schema, Evidence, and memory records share that one collection and are separated by the `kind` payload. |\n", - "", -) -text += "\nWorkspace. Qdrant. Collection. Schema. Evidence. Memory. Payload kind.\n" -path.write_text(text) -PY -set +e -verify_markdown_table_relationships "$local_manual_scattered" "local manual scattered ownership" "Semantic index ownership contract" "$semantic_index_spec" >"$workspace_output" 2>&1 -workspace_status=$? -set -e -if [[ $workspace_status -eq 0 ]]; then - echo "scattered ownership tokens fixture was not rejected correctly" >&2 - cat "$workspace_output" >&2 - exit 1 -fi - -compact_paraphrase="$negative_root/compact-paraphrase.md" -cp "$root/docs/installazione-docker-4-contesti.md" "$compact_paraphrase" -python3 - "$compact_paraphrase" <<'PY' -import pathlib, sys -path = pathlib.Path(sys.argv[1]) -text = path.read_text() -text = text.replace("| DWH | Esterno | Endpoint esterno configurato dall'installazione. |", "| DWH | Esterno | Endpoint esterno deciso dall'installazione. |") -text = text.replace("| LLM | Esterno | Endpoint o policy esterna all'infrastruttura semantica interna. |", "| LLM | Esterno | Endpoint o policy che resta esterna all'infrastruttura semantica interna. |") -text = text.replace("| Qdrant | Interno | Servizio Compose interno obbligatorio con volume persistente `qdrant-data`. |", "| Qdrant | Interno | Servizio Compose interno obbligatorio con il volume persistente `qdrant-data`. |") -text = text.replace("| Ollama embedding | Interno | Servizio Compose interno obbligatorio per `qwen3-embedding:0.6b`. |", "| Ollama embedding | Interno | Servizio Compose interno obbligatorio dedicato a `qwen3-embedding:0.6b`. |") -path.write_text(text) -PY -compact_spec='{"rows":[ - {"componente":"^DWH$","ownership":"^Esterno$","contratto operativo":"endpoint.*estern"}, - {"componente":"^LLM$","ownership":"^Esterno$","contratto operativo":"esterna|esterno"}, - {"componente":"^Qdrant$","ownership":"^Interno$","contratto operativo":"interno.*obbligatorio.*qdrant-data"}, - {"componente":"^Ollama embedding$","ownership":"^Interno$","contratto operativo":"interno.*obbligatorio.*qwen3-embedding:0\\.6b"} -]}' -verify_markdown_table_relationships "$compact_paraphrase" "compact manual paraphrase" "Contratto sintetico di ownership" "$compact_spec" >/dev/null - -compact_inversion="$negative_root/compact-inversion.md" -cp "$root/docs/installazione-docker-4-contesti.md" "$compact_inversion" -python3 - "$compact_inversion" <<'PY' -import pathlib, sys -path = pathlib.Path(sys.argv[1]) -text = path.read_text() -text = text.replace("| Qdrant | Interno | Servizio Compose interno obbligatorio con volume persistente `qdrant-data`. |", "| Qdrant | Esterno | Servizio esterno condiviso. |") -path.write_text(text) -PY -set +e -verify_markdown_table_relationships "$compact_inversion" "compact inversion" "Contratto sintetico di ownership" "$compact_spec" >"$workspace_output" 2>&1 -workspace_status=$? -set -e -if [[ $workspace_status -eq 0 ]]; then - echo "compact inversion fixture was not rejected correctly" >&2 - cat "$workspace_output" >&2 - exit 1 -fi - -compact_scattered="$negative_root/compact-scattered.md" -cp "$root/docs/installazione-docker-4-contesti.md" "$compact_scattered" -python3 - "$compact_scattered" <<'PY' -import pathlib, sys -path = pathlib.Path(sys.argv[1]) -text = path.read_text() -start = text.index("## Contratto sintetico di ownership") -end = text.index("## Comando standard locale") -text = text[:start] + "Qdrant Interno DWH Esterno LLM Esterno Ollama embedding Interno.\n\n" + text[end:] -path.write_text(text) -PY -set +e -verify_markdown_table_relationships "$compact_scattered" "compact scattered tokens" "Contratto sintetico di ownership" "$compact_spec" >"$workspace_output" 2>&1 -workspace_status=$? -set -e -if [[ $workspace_status -eq 0 ]]; then - echo "compact scattered-token fixture was not rejected correctly" >&2 - cat "$workspace_output" >&2 - exit 1 -fi - -adapted_reorder="$negative_root/caddy-adapted-reorder.json" -adapted_bypass="$negative_root/caddy-adapted-bypass.json" -node - "$adapted_reorder" "$adapted_bypass" <<'NODE' -const fs = require("fs"); -const publicHeaders = [ - "X-Thoth-Principal-Issuer", "X-Thoth-Principal-Subject", - "X-Thoth-Principal-Display-Name", "X-Thoth-Is-Admin", -]; -const trustedHeaders = [ - "X-Thoth-Trusted-Principal-Issuer", "X-Thoth-Trusted-Principal-Subject", - "X-Thoth-Trusted-Principal-Display-Name", "X-Thoth-Trusted-Is-Admin", -]; -const clear = (name) => ({handler: "headers", request: {delete: [name]}}); -const auth = { - handler: "reverse_proxy", upstreams: [{dial: "auth-gateway:4180"}], - handle_response: [{match: {status_code: [2]}, routes: [{handle: trustedHeaders.map((name, index) => ({ - handler: "headers", request: {set: {[name]: [`{http.reverse_proxy.header.${publicHeaders[index]}}`]}}, - }))}]}], -}; -const document = {routes: [{handle: [ - ...publicHeaders.map(clear), auth, ...trustedHeaders.map(clear), - {handler: "reverse_proxy", upstreams: [{dial: "127.0.0.1:8080"}]}, -]}]}; -fs.writeFileSync(process.argv[2], JSON.stringify(document)); -const frontend = {handler: "reverse_proxy", upstreams: [{dial: "127.0.0.1:8080"}]}; -const validChain = [...publicHeaders, ...trustedHeaders].map(clear).concat(auth, frontend); -fs.writeFileSync(process.argv[3], JSON.stringify({routes: [ - {handle: validChain}, - {handle: [frontend]}, -]})); -NODE -adapted_output="$negative_root/caddy-adapted-output" -set +e -verify_caddy_adapted_identity_order "$adapted_reorder" >"$adapted_output" 2>&1 -adapted_status=$? -set -e -if [[ $adapted_status -eq 0 ]] || ! grep -Fq "Caddy adapted identity clears must execute before authentication" "$adapted_output"; then - echo "Caddy reordered adapted-handler fixture was not rejected correctly" >&2 - cat "$adapted_output" >&2 - exit 1 -fi - -set +e -verify_caddy_adapted_identity_order "$adapted_bypass" >"$adapted_output" 2>&1 -adapted_status=$? -set -e -if [[ $adapted_status -eq 0 ]] || ! grep -Fq "Caddy adapted frontend path bypasses complete authentication contract" "$adapted_output"; then - echo "Caddy additional direct frontend route fixture was not rejected correctly" >&2 - cat "$adapted_output" >&2 - exit 1 -fi - -negative_failures=0 -expect_evidence_fixture_rejected() { - local label="$1" target="$2" mutation="$3" expected_error="$4" - local fixture_root="$negative_root/evidence-${label// /-}" - local fixture_output="$fixture_root/output" - - # Before Task 7's dedicated verifier exists, every mutation is deliberately accepted. This - # makes the complete Evidence test matrix RED without allowing command-not-found to abort it. - if ! declare -F verify_workspace_evidence_contract >/dev/null; then - echo "negative fixture accepted: $label (Evidence verifier missing)" >&2 - negative_failures=$((negative_failures + 1)) - return - fi - - mkdir -p \ - "$fixture_root/deploy/workspaces" \ - "$fixture_root/docs/contracts" \ - "$fixture_root/docs/install/examples" \ - "$fixture_root/docs/install" \ - "$fixture_root/docs/migrations" - cp "$root/deploy/workspaces/example.yaml" "$fixture_root/deploy/workspaces/example.yaml" - cp "$root/deploy/workspaces/psd.yaml.example" "$fixture_root/deploy/workspaces/psd.yaml.example" - cp "$root/docs/contracts/workspace-evidence-v3.md" \ - "$fixture_root/docs/contracts/workspace-evidence-v3.md" - cp "$root/docs/install/local-workspace-registry.md" \ - "$fixture_root/docs/install/local-workspace-registry.md" - cp "$root/docs/install/server-workspace-registry.md" \ - "$fixture_root/docs/install/server-workspace-registry.md" - cp "$root/README.md" "$fixture_root/README.md" - cp "$root/docs/migrations/p1-to-p1-1-registry-layout.md" \ - "$fixture_root/docs/migrations/p1-to-p1-1-registry-layout.md" - cp "$root/docs/install/examples/workspace-bindings.env.example" \ - "$fixture_root/docs/install/examples/workspace-bindings.env.example" - - python3 - "$fixture_root/$target" "$mutation" <<'PY' -import pathlib, sys, yaml -path = pathlib.Path(sys.argv[1]) -mutation = sys.argv[2] -original = path.read_text() -changed = original -if mutation == "layout-omitted": - changed = original.replace("├── thoth-workspaces.yaml\n", "", 1) -elif mutation == "same-commit-omitted": - changed = original.replace( - "| Revision identity | The catalog blob, descriptor blob, and filesystem Evidence root tree are checked at the same 40-hex Git commit. |\n", - "", - 1, - ) -elif mutation == "flat-descriptor-path": - changed = original.replace("/workspace.yaml", "workspaces/.yaml", 1) -elif mutation in {"absolute-filesystem", "cross-workspace", "old-filesystem-layout"}: - document = yaml.safe_load(original) - document["evidence"]["source"]["uri"] = { - "absolute-filesystem": "/srv/evidence", - "cross-workspace": "other-workspace/evidence", - "old-filesystem-layout": "workspace-content/example/evidence", - }[mutation] - changed = yaml.safe_dump(document, sort_keys=False) -elif mutation == "wrong-docs-directory": - changed = original.replace( - "workspace-docs//{contract.env.example,README.md}", - "example/README.md and example/contract.env.example", - 1, - ) -elif mutation == "catalog-authority-omitted": - changed = original.replace( - "authoritative for workspace ID,", - "descriptor metadata may override workspace ID,", - 1, - ) -elif mutation == "bootstrap-omitted": - changed = original.replace( - "5. The API may create `/workspace.yaml` only when the catalog slot already exists and no Git\n object exists at that path in the exact pulled base commit.\n", - "", - 1, - ) -elif mutation == "api-updates-existing": - changed = original.replace( - "6. After bootstrap, existing descriptors change only through curator Git commit/push and\n installation pull. The API never writes `thoth-workspaces.yaml` or `/evidence/**`.\n", - "6. After bootstrap, use the API to update or delete existing descriptors directly from ThothII.\n", - 1, - ) -elif mutation == "public-http-mode-omitted": - changed = original.replace( - "Public HTTP (`authentication: none`) uses the declared query-free\nURIs directly and requires no Evidence credential file.", - "", - 1, - ) -elif mutation == "ambient-s3-mode-omitted": - changed = original.replace( - "Ambient S3\n(`credentials: ambient`) uses the runtime provider chain and requires no Evidence credential file.", - "", - 1, - ) -elif mutation == "numeric-domains-omitted": - changed = original.replace("positive safe integers", "positive integers", 1) - changed = changed.replace("nonnegative safe integer", "nonnegative integer", 1) -elif mutation == "endpoint-without-url-invariant-omitted": - changed = original.replace( - "Endpoint-policy flags cannot be enabled without `endpoint_url`.", "", 1 - ) -elif mutation == "http-file-boundary-omitted": - changed = original.replace( - "| Signed HTTP | `THT_WS__EVIDENCE_SIGNED_URLS_FILE` | Required for `signed_urls_file`; at most 1048576 bytes; nonempty UTF-8 JSON string array in declared-URI order; query-stripped identities must match `uris` one-to-one. |\n", - "", - 1, - ) -elif mutation == "s3-pair-boundary-omitted": - changed = original.replace( - "| Static S3 pair | `THT_WS__EVIDENCE_ACCESS_KEY_FILE` and `THT_WS__EVIDENCE_SECRET_KEY_FILE` | Required together for `static_files`; each file is at most 65536 bytes. |\n", - "", - 1, - ) -elif mutation == "s3-token-boundary-omitted": - changed = original.replace( - "| Static S3 session | `THT_WS__EVIDENCE_SESSION_TOKEN_FILE` | Optional, valid only with the required access/secret pair, and at most 65536 bytes. |\n", - "", - 1, - ) -elif mutation == "credential-literal": - changed = original + "\nTHT_WS_STATIC_S3_EVIDENCE_SECRET_KEY=AKIAEXAMPLECREDENTIAL\n" -elif mutation == "credential-literal-public-prose": - changed = original + "\nPublic credential example: AKIAABCDEFGHIJKLMNOP\n" -elif mutation == "credential-literal-public-yaml": - document = yaml.safe_load(original) - document["public_credential_example"] = "AKIAABCDEFGHIJKLMNOP" - changed = yaml.safe_dump(document, sort_keys=False) -elif mutation == "signed-query-example": - signed_query = "https://evidence.example.invalid/report" + "?X-Amz-Signature=unsafe" - changed = original + f"\nTHT_EVIDENCE_URI={signed_query}\n" -elif mutation == "unsafe-placeholder": - changed = original.replace( - "/run/secrets/signed-http-evidence-urls.json", "changeme", 1 - ) -elif mutation == "p1-scope-inversion": - changed = original.replace( - "P1.1 performs no acquisition, extraction, preprocessing/indexing, embeddings, Qdrant writes,\nactive-snapshot retention, or GC.", - "P1.1 materializes, extracts, and indexes Evidence before publication.", - 1, - ) -elif mutation.startswith("p1-append-"): - operations = { - "acquisition": ("acquire Evidence", "acquires Evidence", "Evidence acquisition"), - "materialization": ( - "materialize Evidence", "materializes Evidence", "Evidence materialization" - ), - "extraction": ("extract Evidence", "extracts Evidence", "Evidence extraction"), - "preprocessing": ( - "preprocess Evidence", "preprocesses Evidence", "Evidence preprocessing" - ), - "embeddings": ("create embeddings", "creates embeddings", "Evidence embeddings"), - "qdrant-writes": ( - "write embeddings to Qdrant", "writes embeddings to Qdrant", - "Evidence Qdrant writes", - ), - "indexing": ("index Evidence", "indexes Evidence", "Evidence indexing"), - "active": ("publish `ACTIVE`", "publishes `ACTIVE`", "Evidence `ACTIVE` publication"), - "retention": ("retain Evidence", "retains Evidence", "Evidence retention"), - "gc": ("garbage-collect Evidence", "garbage-collects Evidence", "Evidence GC"), - } - forms = ( - "base", "third-person", "can", "may", "must", "will", "should", - "adverb-before-modal", "adverb-after-modal", "ownership", - ) - suffix = mutation.removeprefix("p1-append-") - operation = next((name for name in operations if suffix.startswith(f"{name}-")), None) - form = suffix.removeprefix(f"{operation}-") if operation else "" - if operation is None or form not in forms: - raise SystemExit(f"unknown P1 append mutation: {mutation}") - base, third_person, ownership = operations[operation] - claims = { - "base": f"P1.1 does {base}.", - "third-person": f"P1.1 {third_person}.", - "can": f"P1.1 can {base}.", - "may": f"P1.1 may {base}.", - "must": f"P1.1 must {base}.", - "will": f"P1.1 will {base}.", - "should": f"P1.1 should {base}.", - "adverb-before-modal": f"P1.1 directly may {base}.", - "adverb-after-modal": f"P1.1 may directly {base}.", - "ownership": f"P1.1 owns {ownership}.", - } - changed = original + f"\n{claims[form]}\n" -elif mutation == "config-ordering": - changed = original.replace( - "tht config check -c ", "tht -c config check", 1 - ) -elif mutation == "acceptance-conflation": - changed = original.replace("manual acceptance: PENDING\n", "", 1) -elif mutation == "curator-order": - second = "2. Keep `thoth-workspaces.yaml` curator-owned. It uses the `schema_version` value `1` and the ordered\n `workspaces` list of `{id, name, description?}` entries; it is authoritative for workspace ID,\n name, description, and display order." - third = "3. For an existing workspace, edit `/workspace.yaml` and any embedded `/evidence/**`, then\n commit and push." - changed = original.replace(second + "\n" + third, third + "\n" + second, 1) -elif mutation == "migration-commit-omitted": - changed = original.replace("git mv workspaces/.yaml /workspace.yaml\n", "", 1) -elif mutation == "migration-upgrade-omitted": - changed = original.replace("3. Upgrade ThothII only after that migration commit is pushed.\n", "", 1) -elif mutation == "migration-rollback-omitted": - changed = original.replace("Roll back the application revision and registry commit together.", "Roll back only the application revision.", 1) -else: - raise SystemExit(f"unknown Evidence mutation: {mutation}") -if changed == original: - raise SystemExit(f"Evidence mutation made no change: {mutation}") -path.write_text(changed) -PY - - set +e - verify_workspace_evidence_contract "$fixture_root" >"$fixture_output" 2>&1 - local status=$? - set -e - if [[ $status -eq 0 ]]; then - echo "negative fixture accepted: $label" >&2 - cat "$fixture_output" >&2 - negative_failures=$((negative_failures + 1)) - elif ! grep -Fq -- "$expected_error" "$fixture_output"; then - echo "negative fixture failed for the wrong reason: $label" >&2 - cat "$fixture_output" >&2 - negative_failures=$((negative_failures + 1)) - fi -} - -expect_evidence_claim_accepted() { - local label="$1" claim="$2" - local fixture_root="$negative_root/evidence-safe-${label// /-}" - local fixture_output="$fixture_root/output" - - mkdir -p \ - "$fixture_root/deploy/workspaces" \ - "$fixture_root/docs/contracts" \ - "$fixture_root/docs/install/examples" \ - "$fixture_root/docs/install" \ - "$fixture_root/docs/migrations" - cp "$root/deploy/workspaces/example.yaml" "$fixture_root/deploy/workspaces/example.yaml" - cp "$root/deploy/workspaces/psd.yaml.example" "$fixture_root/deploy/workspaces/psd.yaml.example" - cp "$root/docs/contracts/workspace-evidence-v3.md" \ - "$fixture_root/docs/contracts/workspace-evidence-v3.md" - cp "$root/docs/install/local-workspace-registry.md" \ - "$fixture_root/docs/install/local-workspace-registry.md" - cp "$root/docs/install/server-workspace-registry.md" \ - "$fixture_root/docs/install/server-workspace-registry.md" - cp "$root/README.md" "$fixture_root/README.md" - cp "$root/docs/migrations/p1-to-p1-1-registry-layout.md" \ - "$fixture_root/docs/migrations/p1-to-p1-1-registry-layout.md" - cp "$root/docs/install/examples/workspace-bindings.env.example" \ - "$fixture_root/docs/install/examples/workspace-bindings.env.example" - printf '\n%s\n' "$claim" >>"$fixture_root/docs/contracts/workspace-evidence-v3.md" - - set +e - verify_workspace_evidence_contract "$fixture_root" >"$fixture_output" 2>&1 - local status=$? - set -e - if [[ $status -ne 0 ]]; then - echo "safe Evidence fixture rejected: $label" >&2 - cat "$fixture_output" >&2 - negative_failures=$((negative_failures + 1)) - fi -} - -expect_guide_rejected() { - local label="$1" validator="$2" source_guide="$3" relative_path="$4" - local mutation="$5" expected_error="$6" - local fixture_root="$negative_root/${label// /-}" - local fixture_output="$fixture_root/output" - mkdir -p "$fixture_root/$(dirname "$relative_path")" - cp "$source_guide" "$fixture_root/$relative_path" - if [[ "$validator" == verify_windows_line_endings_guide ]]; then - mkdir -p "$fixture_root/scripts" - cp "$root/scripts/verify-line-endings.sh" "$fixture_root/scripts/verify-line-endings.sh" - elif [[ "$validator" == verify_pi_management_guide ]]; then - mkdir -p "$fixture_root/docs/contracts" - cp "$root/docs/contracts/tht-pi.md" "$fixture_root/docs/contracts/tht-pi.md" - fi - node - "$fixture_root/$relative_path" "$mutation" <<'NODE' -const fs = require("fs"); -const [path, mutation] = process.argv.slice(2); -const original = fs.readFileSync(path, "utf8"); -let changed = original; -switch (mutation) { - case "durable-selector": - changed = original.replaceAll("--source build", "--source stale-build"); - break; - case "dangerous-volumes": - changed = original.replace("Do **not** run `docker compose down --volumes`", "Run `docker compose down --volumes`"); - break; - case "incomplete-powershell": - changed = original.replaceAll("icacls.exe", "Write-Output"); - break; - case "broken-crlf": - changed = original.replaceAll("git checkout-index --all --force --prefix=", "git add --renormalize . # "); - break; - case "raw-pi": - changed += "\n```sh\ndocker compose exec core pi --version\n```\n"; - break; - case "server-secret-env": - changed += "\n```dotenv\nTHT_MODEL_API_KEY=unsafe-secret-value\n```\n"; - break; - case "server-docker-socket": - changed += "\nMount /var/run/docker.sock into core for management.\n"; - break; - case "server-coupling": - changed += "\nAttach core to the omics_portal application network.\n"; - break; - case "server-host-loopback": - changed += "\nFor host-gateway, keep the external service listening on 127.0.0.1.\n"; - break; - case "server-parent-traversal": - changed = original.replace('sudo install -d -o "$operator_uid" -g 10001 -m 0750 /srv/thothii\n', ''); - break; - case "server-host-account": - changed += "\n```sh\nsudo useradd --system --uid 10001 thothii\n```\n"; - break; - case "server-raw-remove": - changed += "\n```sh\ndocker rm thothii-core thothii-frontend\n```\n"; - break; - case "server-pinned-migrator-mismatch": - changed += "\n```yaml\nservices:\n core:\n image: registry.invalid/core@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\n session-migrate:\n image: thothii-core:local\n```\n"; - break; - case "server-pinned-frontend-missing": - changed = original.replace(' frontend:\n build: !reset null\n image: registry.example.com/thothii/frontend@sha256:<64-lowercase-hex-digits>\n', ''); - break; - case "nginx-no-auth": - changed = original.replace(" auth_request /_authenticate;", " # authentication omitted"); - break; - case "nginx-core-upstream": - changed = original.replaceAll("http://127.0.0.1:8080", "http://127.0.0.1:8787"); - break; - case "nginx-no-sse": - changed = original.replace(" proxy_buffering off;", " proxy_buffering on;"); - break; - case "nginx-no-issuer-clear": - changed = original.replaceAll('proxy_set_header X-Thoth-Principal-Issuer "";', 'proxy_set_header X-Thoth-Principal-Issuer $http_x_thoth_principal_issuer;'); - break; - case "nginx-no-subject-capture": - changed = original.replace("auth_request_set $thoth_principal_subject", "# missing auth capture $thoth_principal_subject"); - break; - case "nginx-no-display-map": - changed = original.replace("proxy_set_header X-Thoth-Trusted-Principal-Display-Name $thoth_principal_display_name;", "proxy_set_header X-Thoth-Trusted-Principal-Display-Name \"\";"); - break; - case "nginx-no-admin-map": - changed = original.replace("proxy_set_header X-Thoth-Trusted-Is-Admin $thoth_is_admin;", "proxy_set_header X-Thoth-Trusted-Is-Admin \"\";"); - break; - case "nginx-admin-clear-wrong-scope": { - const clear = ' proxy_set_header X-Thoth-Is-Admin "";'; - const authAt = original.indexOf(clear); - changed = original.slice(0, authAt) + original.slice(authAt + clear.length + 1); - const frontendAt = changed.indexOf(clear); - changed = changed.slice(0, frontendAt) + clear + "\n" + clear + changed.slice(frontendAt + clear.length); - break; - } - case "nginx-additional-bypass": - changed = original.replace(" location / {", " location /bypass {\n proxy_pass http://127.0.0.1:8080;\n }\n\n location / {"); - break; - case "nginx-comment-only-auth": - changed = original.replace(" location / {", ` location /comment-only-auth { - # auth_request /_authenticate; - # auth_request_set $thoth_principal_issuer $upstream_http_x_thoth_principal_issuer; - # auth_request_set $thoth_principal_subject $upstream_http_x_thoth_principal_subject; - # auth_request_set $thoth_principal_display_name $upstream_http_x_thoth_principal_display_name; - # auth_request_set $thoth_is_admin $upstream_http_x_thoth_is_admin; - # proxy_set_header X-Thoth-Principal-Issuer ""; - # proxy_set_header X-Thoth-Principal-Subject ""; - # proxy_set_header X-Thoth-Principal-Display-Name ""; - # proxy_set_header X-Thoth-Is-Admin ""; - # proxy_set_header X-Thoth-Trusted-Principal-Issuer $thoth_principal_issuer; - # proxy_set_header X-Thoth-Trusted-Principal-Subject $thoth_principal_subject; - # proxy_set_header X-Thoth-Trusted-Principal-Display-Name $thoth_principal_display_name; - # proxy_set_header X-Thoth-Trusted-Is-Admin $thoth_is_admin; - proxy_set_header X-Comment-Literal "quoted#value"; # preserve the quoted hash - proxy_pass http://127.0.0.1:8080; # active frontend path - } - - location / {`); - break; - case "caddy-no-auth": - changed = original.replace("forward_auth auth-gateway:4180 {", "# forward authentication omitted"); - break; - case "caddy-client-identity": - changed = original.replace("X-Thoth-Principal-Subject>X-Thoth-Trusted-Principal-Subject", "X-Thoth-Principal-Subject"); - break; - case "caddy-core-upstream": - changed = original.replaceAll("127.0.0.1:8080", "127.0.0.1:8787"); - break; - case "caddy-no-issuer-public-clear": - changed = original.replace("request_header -X-Thoth-Principal-Issuer", "request_header X-Thoth-Principal-Issuer {header.X-Thoth-Principal-Issuer}"); - break; - case "caddy-no-subject-trusted-clear": - changed = original.replace("request_header -X-Thoth-Trusted-Principal-Subject", "request_header X-Thoth-Trusted-Principal-Subject {header.X-Thoth-Trusted-Principal-Subject}"); - break; - case "caddy-no-display-map": - changed = original.replace("X-Thoth-Principal-Display-Name>X-Thoth-Trusted-Principal-Display-Name", "X-Thoth-Principal-Display-Name"); - break; - case "caddy-no-admin-map": - changed = original.replace("X-Thoth-Is-Admin>X-Thoth-Trusted-Is-Admin", "X-Thoth-Is-Admin"); - break; - case "caddy-clears-after-auth": { - const clearPattern = /(?:\t\trequest_header -X-(?:Authenticated-User|Thoth-[^\n]+)\n)+/; - const clears = original.match(clearPattern)?.[0] || ""; - changed = original.replace(clearPattern, ""); - changed = changed.replace("\n\t\treverse_proxy 127.0.0.1:8080 {", "\n" + clears + "\n\t\treverse_proxy 127.0.0.1:8080 {"); - break; - } - case "dirty-source": - changed = original.replaceAll("git status --porcelain --untracked-files=all", "git status --short"); - break; - case "failed-pull": - changed = original.replace("if ! git pull --ff-only; then abort_update", "if git pull --ff-only; then abort_update"); - break; - case "failed-status": - changed = original.replace("if ! RUNNING_PI_VERSION=", "if RUNNING_PI_VERSION="); - break; - case "failed-build": - changed = original.replace("if ! bash scripts/build-local.sh; then", "if bash scripts/build-local.sh; then"); - break; - case "same-version-no-selector": - changed = original.replace("TRANSACTIONAL_PI_UPDATE=false", "TRANSACTIONAL_PI_UPDATE=true # unsafe same-version no-op"); - break; - case "powershell-source-failure": - changed = original.replace("Assert-NativeSuccess 'Pi status'", "Write-Output 'Pi status unchecked'"); - break; - case "failed-export": - changed = original.replace("if ! git checkout-index --all --force", "if git checkout-index --all --force"); - break; - case "partial-export": - changed = original.replace("if ! validate_index_export; then", "if validate_index_export; then"); - break; - case "mode-120000": - changed = original.replaceAll("120000", "100644-no-symlink-mode"); - break; - case "powershell-crlf-failure": - changed = original.replace("Assert-NativeSuccess 'index export'", "Write-Output 'index export unchecked'"); - break; - default: - throw new Error(`unknown negative-fixture mutation: ${mutation}`); -} -if (changed === original) throw new Error(`negative-fixture mutation made no change: ${mutation}`); -fs.writeFileSync(path, changed); -NODE - set +e - (root="$fixture_root"; set -e; "$validator") >"$fixture_output" 2>&1 - local status=$? - set -e - if [[ $status -eq 0 ]]; then - echo "negative fixture accepted: $label" >&2 - cat "$fixture_output" >&2 - negative_failures=$((negative_failures + 1)) - elif ! grep -Fq -- "$expected_error" "$fixture_output"; then - echo "negative fixture failed for the wrong reason: $label" >&2 - cat "$fixture_output" >&2 - negative_failures=$((negative_failures + 1)) - fi -} - -expect_guide_rejected \ - "durable selector keeps old core" verify_local_guide \ - "$root/docs/install/local.md" docs/install/local.md durable-selector \ - "installation-aware source update lacks structural token: --source build" -expect_guide_rejected \ - "dangerous down volumes instruction" verify_local_guide \ - "$root/docs/install/local.md" docs/install/local.md dangerous-volumes \ - "docker compose down --volumes must appear only in an explicit prose prohibition" -expect_guide_rejected \ - "incomplete native PowerShell path" verify_local_guide \ - "$root/docs/install/local.md" docs/install/local.md incomplete-powershell \ - "native PowerShell setup lacks structural token: icacls.exe" -expect_guide_rejected \ - "renormalize leaves CRLF worktree bytes" verify_windows_line_endings_guide \ - "$root/docs/install/windows-line-endings.md" docs/install/windows-line-endings.md broken-crlf \ - "Windows line-ending guide lacks required instruction: git checkout-index --all --force" -expect_guide_rejected \ - "raw non-installation-aware Pi access" verify_pi_management_guide \ - "$root/docs/install/pi-management.md" docs/install/pi-management.md raw-pi \ - "raw non-installation-aware Compose Pi access is forbidden" -expect_guide_rejected \ - "server secret in environment" verify_server_guide \ - "$root/docs/install/server.md" docs/install/server.md server-secret-env \ - "server installation guide embeds a secret value" -expect_guide_rejected \ - "server Docker socket mount" verify_server_guide \ - "$root/docs/install/server.md" docs/install/server.md server-docker-socket \ - "server installation guide introduces a Docker socket dependency" -expect_guide_rejected \ - "server application coupling" verify_server_guide \ - "$root/docs/install/server.md" docs/install/server.md server-coupling \ - "server installation guide introduces forbidden application coupling" -expect_guide_rejected \ - "server host-gateway loopback listener" verify_server_guide \ - "$root/docs/install/server.md" docs/install/server.md server-host-loopback \ - "server host-gateway guidance assumes a host loopback listener" -expect_guide_rejected \ - "server parent traversal boundary" verify_server_guide \ - "$root/docs/install/server.md" docs/install/server.md server-parent-traversal \ - "server installation guide does not set parent traversal boundary" -expect_guide_rejected \ - "server host account creation" verify_server_guide \ - "$root/docs/install/server.md" docs/install/server.md server-host-account \ - "server installation guide creates or depends on a host identity" -expect_guide_rejected \ - "server raw container removal" verify_server_guide \ - "$root/docs/install/server.md" docs/install/server.md server-raw-remove \ - "server uninstall bypasses installation-aware removal" -expect_guide_rejected \ - "server pinned migrator differs from core" verify_server_guide \ - "$root/docs/install/server.md" docs/install/server.md server-pinned-migrator-mismatch \ - "server pinned migration image must equal the pinned core image" -expect_guide_rejected \ - "server pinned frontend is missing" verify_server_guide \ - "$root/docs/install/server.md" docs/install/server.md server-pinned-frontend-missing \ - "server pinned image override must pin core, session-migrate, and frontend without builds" -expect_guide_rejected \ - "Nginx identity without authentication" verify_reverse_proxy_nginx_guide \ - "$root/docs/install/reverse-proxy-nginx.md" docs/install/reverse-proxy-nginx.md nginx-no-auth \ - "Nginx proxy lacks structural token: auth_request /_authenticate;" -expect_guide_rejected \ - "Nginx direct core exposure" verify_reverse_proxy_nginx_guide \ - "$root/docs/install/reverse-proxy-nginx.md" docs/install/reverse-proxy-nginx.md nginx-core-upstream \ - "Nginx proxy must forward only to frontend on 127.0.0.1:8080" -expect_guide_rejected \ - "Nginx buffered SSE" verify_reverse_proxy_nginx_guide \ - "$root/docs/install/reverse-proxy-nginx.md" docs/install/reverse-proxy-nginx.md nginx-no-sse \ - "Nginx proxy lacks structural token: proxy_buffering off;" -expect_guide_rejected \ - "Nginx issuer inbound claim not cleared" verify_reverse_proxy_nginx_guide \ - "$root/docs/install/reverse-proxy-nginx.md" docs/install/reverse-proxy-nginx.md nginx-no-issuer-clear \ - "Nginx auth location does not clear inbound issuer identity" -expect_guide_rejected \ - "Nginx subject auth response not captured" verify_reverse_proxy_nginx_guide \ - "$root/docs/install/reverse-proxy-nginx.md" docs/install/reverse-proxy-nginx.md nginx-no-subject-capture \ - "Nginx frontend location does not capture authenticated subject identity" -expect_guide_rejected \ - "Nginx display identity not mapped to private hop" verify_reverse_proxy_nginx_guide \ - "$root/docs/install/reverse-proxy-nginx.md" docs/install/reverse-proxy-nginx.md nginx-no-display-map \ - "Nginx frontend location does not map authenticated display identity" -expect_guide_rejected \ - "Nginx admin identity not mapped to private hop" verify_reverse_proxy_nginx_guide \ - "$root/docs/install/reverse-proxy-nginx.md" docs/install/reverse-proxy-nginx.md nginx-no-admin-map \ - "Nginx frontend location does not map authenticated admin identity" -expect_guide_rejected \ - "Nginx admin clear moved out of auth scope" verify_reverse_proxy_nginx_guide \ - "$root/docs/install/reverse-proxy-nginx.md" docs/install/reverse-proxy-nginx.md nginx-admin-clear-wrong-scope \ - "Nginx auth location does not clear inbound admin identity" -expect_guide_rejected \ - "Nginx additional frontend bypass location" verify_reverse_proxy_nginx_guide \ - "$root/docs/install/reverse-proxy-nginx.md" docs/install/reverse-proxy-nginx.md nginx-additional-bypass \ - "Nginx direct OIDC mode contains an additional frontend bypass location" -expect_guide_rejected \ - "Nginx frontend auth directives only in comments" verify_reverse_proxy_nginx_guide \ - "$root/docs/install/reverse-proxy-nginx.md" docs/install/reverse-proxy-nginx.md nginx-comment-only-auth \ - "Nginx direct OIDC mode contains an additional frontend bypass location" -expect_guide_rejected \ - "Caddy identity without authentication" verify_reverse_proxy_caddy_guide \ - "$root/docs/install/reverse-proxy-caddy.md" docs/install/reverse-proxy-caddy.md caddy-no-auth \ - "Caddy proxy lacks structural token: forward_auth auth-gateway:4180 {" -expect_guide_rejected \ - "Caddy untrusted identity forwarding" verify_reverse_proxy_caddy_guide \ - "$root/docs/install/reverse-proxy-caddy.md" docs/install/reverse-proxy-caddy.md caddy-client-identity \ - "Caddy proxy does not map authenticated subject identity" -expect_guide_rejected \ - "Caddy direct core exposure" verify_reverse_proxy_caddy_guide \ - "$root/docs/install/reverse-proxy-caddy.md" docs/install/reverse-proxy-caddy.md caddy-core-upstream \ - "Caddy proxy must forward only to frontend on 127.0.0.1:8080" -expect_guide_rejected \ - "Caddy issuer inbound claim not cleared" verify_reverse_proxy_caddy_guide \ - "$root/docs/install/reverse-proxy-caddy.md" docs/install/reverse-proxy-caddy.md caddy-no-issuer-public-clear \ - "Caddy proxy does not clear inbound issuer identity" -expect_guide_rejected \ - "Caddy subject private-hop claim not cleared" verify_reverse_proxy_caddy_guide \ - "$root/docs/install/reverse-proxy-caddy.md" docs/install/reverse-proxy-caddy.md caddy-no-subject-trusted-clear \ - "Caddy proxy does not clear inbound trusted subject identity" -expect_guide_rejected \ - "Caddy display identity not mapped to private hop" verify_reverse_proxy_caddy_guide \ - "$root/docs/install/reverse-proxy-caddy.md" docs/install/reverse-proxy-caddy.md caddy-no-display-map \ - "Caddy proxy does not map authenticated display identity" -expect_guide_rejected \ - "Caddy admin identity not mapped to private hop" verify_reverse_proxy_caddy_guide \ - "$root/docs/install/reverse-proxy-caddy.md" docs/install/reverse-proxy-caddy.md caddy-no-admin-map \ - "Caddy proxy does not map authenticated admin identity" -expect_guide_rejected \ - "Caddy identity clears reordered after auth" verify_reverse_proxy_caddy_guide \ - "$root/docs/install/reverse-proxy-caddy.md" docs/install/reverse-proxy-caddy.md caddy-clears-after-auth \ - "Caddy identity clears must precede forward_auth" - -expect_guide_rejected \ - "dirty or untracked source tree" verify_local_guide \ - "$root/docs/install/local.md" docs/install/local.md dirty-source \ - "installation-aware source update lacks structural token: git status --porcelain --untracked-files=all" -expect_guide_rejected \ - "failed source pull" verify_local_guide \ - "$root/docs/install/local.md" docs/install/local.md failed-pull \ - "POSIX source update does not fail closed: source pull" -expect_guide_rejected \ - "failed tht Pi status" verify_local_guide \ - "$root/docs/install/local.md" docs/install/local.md failed-status \ - "POSIX source update does not fail closed: Pi status" -expect_guide_rejected \ - "failed local build" verify_local_guide \ - "$root/docs/install/local.md" docs/install/local.md failed-build \ - "POSIX source update does not fail closed: local build" -expect_guide_rejected \ - "same Pi version without durable selector" verify_local_guide \ - "$root/docs/install/local.md" docs/install/local.md same-version-no-selector \ - "POSIX source update lacks the same-version/no-selector path" -expect_guide_rejected \ - "PowerShell source command failure propagation" verify_local_guide \ - "$root/docs/install/local.md" docs/install/local.md powershell-source-failure \ - "PowerShell source update does not propagate failure: Pi status" -expect_guide_rejected \ - "failed index export" verify_windows_line_endings_guide \ - "$root/docs/install/windows-line-endings.md" docs/install/windows-line-endings.md failed-export \ - "POSIX CRLF repair lacks fail-closed semantic: if ! git checkout-index" -expect_guide_rejected \ - "partial index export" verify_windows_line_endings_guide \ - "$root/docs/install/windows-line-endings.md" docs/install/windows-line-endings.md partial-export \ - "POSIX CRLF repair does not prove a complete export before destructive rewrite" -expect_guide_rejected \ - "mode 120000 symlink preservation" verify_windows_line_endings_guide \ - "$root/docs/install/windows-line-endings.md" docs/install/windows-line-endings.md mode-120000 \ - "POSIX CRLF repair lacks fail-closed semantic: 120000" -expect_guide_rejected \ - "PowerShell CRLF command failure propagation" verify_windows_line_endings_guide \ - "$root/docs/install/windows-line-endings.md" docs/install/windows-line-endings.md powershell-crlf-failure \ - "PowerShell CRLF repair lacks failure propagation: Assert-NativeSuccess 'index export'" - -expect_evidence_fixture_rejected "root catalog omitted" docs/contracts/workspace-evidence-v3.md layout-omitted "missing canonical Evidence layout" -expect_evidence_fixture_rejected "same revision ownership omitted" docs/contracts/workspace-evidence-v3.md same-commit-omitted "missing same-revision ownership" -expect_evidence_fixture_rejected "flat descriptor path" docs/contracts/workspace-evidence-v3.md flat-descriptor-path 'The descriptor at `/workspace.yaml` must match the' -expect_evidence_fixture_rejected "absolute filesystem Evidence path" deploy/workspaces/example.yaml absolute-filesystem "noncanonical filesystem Evidence URI" -expect_evidence_fixture_rejected "cross-workspace Evidence path" deploy/workspaces/psd.yaml.example cross-workspace "Evidence namespace mismatch" -expect_evidence_fixture_rejected "old filesystem Evidence layout" deploy/workspaces/example.yaml old-filesystem-layout "Evidence namespace mismatch" -expect_evidence_fixture_rejected "public HTTP mode omitted" docs/contracts/workspace-evidence-v3.md public-http-mode-omitted "missing public HTTP mode" -expect_evidence_fixture_rejected "ambient S3 mode omitted" docs/contracts/workspace-evidence-v3.md ambient-s3-mode-omitted "missing ambient S3 mode" -expect_evidence_fixture_rejected "strict Evidence numeric domains omitted" docs/contracts/workspace-evidence-v3.md numeric-domains-omitted "missing strict Evidence numeric domains" -expect_evidence_fixture_rejected "S3 endpoint policy without endpoint invariant omitted" docs/contracts/workspace-evidence-v3.md endpoint-without-url-invariant-omitted "missing S3 endpoint policy without endpoint invariant" -expect_evidence_fixture_rejected "signed HTTP file boundary omitted" docs/contracts/workspace-evidence-v3.md http-file-boundary-omitted "missing signed HTTP file boundary" -expect_evidence_fixture_rejected "static S3 pair boundary omitted" docs/contracts/workspace-evidence-v3.md s3-pair-boundary-omitted "missing static S3 file boundary" -expect_evidence_fixture_rejected "static S3 optional token boundary omitted" docs/contracts/workspace-evidence-v3.md s3-token-boundary-omitted "missing static S3 session-token boundary" -expect_evidence_fixture_rejected "credential literal in public bindings" docs/install/examples/workspace-bindings.env.example credential-literal "credential literal forbidden" -expect_evidence_fixture_rejected "credential literal in public prose" docs/contracts/workspace-evidence-v3.md credential-literal-public-prose "credential literal forbidden" -expect_evidence_fixture_rejected "credential literal in public YAML" deploy/workspaces/example.yaml credential-literal-public-yaml "credential literal forbidden" -expect_evidence_fixture_rejected "signed query in public bindings" docs/install/examples/workspace-bindings.env.example signed-query-example "query-bearing public URI forbidden" -expect_evidence_fixture_rejected "unsafe Evidence file placeholder" docs/install/examples/workspace-bindings.env.example unsafe-placeholder "unsafe file placeholder/path" -expect_evidence_fixture_rejected "P1.1 Evidence scope inversion" docs/contracts/workspace-evidence-v3.md p1-scope-inversion "P1.1 scope violation" -expect_evidence_fixture_rejected "migration commit step omitted" docs/migrations/p1-to-p1-1-registry-layout.md migration-commit-omitted "migration guide missing commit step" -expect_evidence_fixture_rejected "migration upgrade ordering omitted" docs/migrations/p1-to-p1-1-registry-layout.md migration-upgrade-omitted "migration guide missing upgrade ordering" -expect_evidence_fixture_rejected "migration rollback rule omitted" docs/migrations/p1-to-p1-1-registry-layout.md migration-rollback-omitted "migration guide missing rollback rule" -p1_operations=( - acquisition materialization extraction preprocessing embeddings - qdrant-writes indexing active retention gc -) -p1_positive_forms=( - base third-person can may must will should - adverb-before-modal adverb-after-modal ownership -) -for operation in "${p1_operations[@]}"; do - for form in "${p1_positive_forms[@]}"; do - expect_evidence_fixture_rejected \ - "appended P1.1 ${operation} ${form} claim" \ - docs/contracts/workspace-evidence-v3.md "p1-append-${operation}-${form}" \ - "P1.1 scope violation" - done -done -p1_safe_bases=( - "acquire Evidence" "materialize Evidence" "extract Evidence" "preprocess Evidence" - "create embeddings" "write embeddings to Qdrant" "index Evidence" 'publish `ACTIVE`' - "retain Evidence" "garbage-collect Evidence" -) -p1_safe_third_person=( - "acquires Evidence" "materializes Evidence" "extracts Evidence" "preprocesses Evidence" - "creates embeddings" "writes embeddings to Qdrant" "indexes Evidence" 'publishes `ACTIVE`' - "retains Evidence" "garbage-collects Evidence" -) -p1_safe_ownership=( - "Evidence acquisition" "Evidence materialization" "Evidence extraction" - "Evidence preprocessing" "Evidence embeddings" "Evidence Qdrant writes" - "Evidence indexing" 'Evidence `ACTIVE` publication' "Evidence retention" "Evidence GC" -) -for index in "${!p1_operations[@]}"; do - operation="${p1_operations[$index]}" - base="${p1_safe_bases[$index]}" - third_person="${p1_safe_third_person[$index]}" - ownership="${p1_safe_ownership[$index]}" - expect_evidence_claim_accepted "negative P1 ${operation} cannot" "P1 cannot ${base}." - expect_evidence_claim_accepted "negative P1 ${operation} must not" "P1 must not ${base}." - expect_evidence_claim_accepted "negative P1 ${operation} does not" "P1 does not ${base}." - expect_evidence_claim_accepted "negative P1 ${operation} never" "P1 never ${third_person}." - expect_evidence_claim_accepted \ - "later plan ${operation}" "A later plan may assign ${ownership} to P1." - expect_evidence_claim_accepted "P2 ${operation}" "P2 may directly ${base}." - expect_evidence_claim_accepted "P6 ${operation}" "P6 ${third_person}." -done -expect_evidence_fixture_rejected \ - "config check option reordered" docs/contracts/workspace-evidence-v3.md config-ordering \ - "exact config-check ordering missing" -expect_evidence_fixture_rejected \ - "acceptance states conflated" docs/contracts/workspace-evidence-v3.md acceptance-conflation \ - "separate automated/manual states missing" - -if (( negative_failures != 0 )); then - echo "$negative_failures unsafe installation-document fixtures were accepted" >&2 - exit 1 -fi - -# Projected server authentication documents must preserve the root-only canonical/runtime split. -projection_example="$root/docs/install/examples/thothii-installation.server.yaml" -for required in \ - 'runtimeProjection:' \ - 'directory: "/srv/example/thothii/auth-runtime"' \ - 'uid: 10001' \ - 'gid: 10001'; do - grep -Fq -- "$required" "$projection_example" || { - echo "server authentication projection example lacks: $required" >&2 - exit 1 - } -done -if rg -n -i --pcre2 '(?:password|secret)[[:space:]]*:[[:space:]]*[^<#[:space:]]+' "$projection_example"; then - echo "server authentication projection example contains a credential value" >&2 - exit 1 -fi - -projection_docs=( - "$root/docs/install/server.md" - "$root/docs/install/authentication-local.md" - "$root/docs/testing/authentication-manual-acceptance.md" - "$root/docs/testing/psd-server-project-a-manual.md" - "$root/docs/plans/2026-08-20-psd-server-project-a-standalone.md" - "$root/PROJECT_STATE.md" -) -projection_corpus="$negative_root/projection-corpus.md" -cat "${projection_docs[@]}" >"$projection_corpus" - -projection_documentation_is_safe() { - local corpus="$1" - if rg -q -i --pcre2 '(?:useradd|groupadd)[^\n]{0,100}10001|10001[^\n]{0,100}(?:useradd|groupadd)' "$corpus"; then - return 1 - fi - if rg -q -i --pcre2 '(?:^|[.!?]\s+)(?:(?:mount|bind)\s+(?:the\s+)?(?:canonical authentication root|auth-canonical)\b[^\n]{0,120}(?:core|/run/thothii-auth)|(?:core|the core service)\s+(?:mounts?|binds?)\s+(?:the\s+)?(?:canonical authentication root|auth-canonical)\b)|auth-canonical[^\n]{0,120}:/run/thothii-auth' "$corpus"; then - return 1 - fi - if rg -q -i --pcre2 '(?:(?:operators?\s+)?(?:may|can|should|must)\s+(?!not\b|never\b)(?:edit|write|modify|prune)|^\s*(?:sudo\s+)?(?:vi|vim|nano|sed|cp|mv|rm|tee|printf|echo)\b)[^\n]*(?:CURRENT|generations)' "$corpus"; then - return 1 - fi - if rg -q -i --pcre2 '(?:^\s*(?:sudo\s+nginx\s+-T|(?:sudo\s+)?(?:printenv|env)(?:\s|$)|(?:sudo\s+)?(?:diff|cat|less|more|head|tail|yq)\s+[^\n]*(?:secret|auth\.yaml|users\.yaml))|(?:may|can|should|must)\s+(?!not\b|never\b)(?:dump|print|capture|attach|include|run)\b[^\n]*(?:password|auth\.yaml|users\.yaml|YAML|raw environment|nginx\s+-T|secret-bearing diff))' "$corpus"; then - return 1 - fi - if rg -q -i --pcre2 'Project A (?:has been|was|is) started|legacy[- ]stack (?:has been|was|is) changed' "$corpus"; then - return 1 - fi - return 0 -} - -for required in \ - 'canonical authentication root' \ - 'read-only and core-only' \ - 'candidate or recovery' \ - 'runtime projection is blocked' \ - 'Project A has not been started'; do - if ! rg -Fqi -- "$required" "${projection_docs[@]}"; then - echo "server authentication projection documentation lacks: $required" >&2 - exit 1 - fi -done -if ! projection_documentation_is_safe "$projection_corpus"; then - echo "server authentication projection documentation contains an unsafe instruction or claim" >&2 - exit 1 -fi - -while IFS='|' read -r fixture_name payload; do - fixture="$negative_root/projection-$fixture_name.md" - cp "$projection_corpus" "$fixture" - printf '\n%s\n' "$payload" >>"$fixture" - if projection_documentation_is_safe "$fixture"; then - echo "server authentication projection documentation accepted unsafe fixture: $fixture_name" >&2 - exit 1 - fi -done <<'PROJECTION_NEGATIVE_FIXTURES' -host-identity|sudo useradd --system --uid 10001 thothii -canonical-core-mount|Mount the canonical authentication root into core at /run/thothii-auth. -core-subject-canonical-mount|Core mounts the canonical authentication root at /run/thothii-auth. -direct-runtime-edit|Operators may edit CURRENT and prune generations directly. -yaml-dump|Operators may dump auth.yaml and users.yaml into evidence. -nginx-dump|sudo nginx -T -raw-environment|printenv -sudo-raw-environment|sudo printenv -secret-diff|sudo diff auth.yaml auth.yaml.previous -live-claim|Project A has been started. -PROJECTION_NEGATIVE_FIXTURES - -echo "unsafe installation-document fixtures rejected passed" +echo 'workspace installation documentation regression suite passed' diff --git a/scripts/verify-workspace-install-docs.sh b/scripts/verify-workspace-install-docs.sh index 0db63f34..5691b9c6 100755 --- a/scripts/verify-workspace-install-docs.sh +++ b/scripts/verify-workspace-install-docs.sh @@ -1,2325 +1,155 @@ #!/usr/bin/env bash -# Verify canonical local/server installation manuals and their base+override Compose paths. +# Verify the public installation and workspace documentation against the current topology. set -euo pipefail -root="$(cd "$(dirname "$0")/.." && pwd -P)" -mode="${1:-}" +if [[ -n "${THT_DOCS_VERIFY_ROOT:-}" ]]; then + root="$(cd "$THT_DOCS_VERIFY_ROOT" && pwd -P)" +else + root="$(cd "$(dirname "$0")/.." && pwd -P)" +fi -tmp_prefix="${TMPDIR:-/tmp}" -while [[ "$tmp_prefix" != "/" && "$tmp_prefix" == */ ]]; do - tmp_prefix="${tmp_prefix%/}" -done -tmp_prefix="${tmp_prefix%/}/" - -trim() { - local value="$1" - value="${value#"${value%%[![:space:]]*}"}" - value="${value%"${value##*[![:space:]]}"}" - printf '%s' "$value" +fail() { + echo "verify-workspace-install-docs.sh: $*" >&2 + exit 1 } -is_safe_absolute_path() { - local value="$1" segment - local -a segments - [[ "$value" == /* && "$value" != *//* ]] || return 1 - IFS=/ read -r -a segments <<<"$value" - for segment in "${segments[@]}"; do - [[ "$segment" != . && "$segment" != .. ]] || return 1 - done -} - -verify_path_variable_values() { - local source="$1" line trimmed name value - while IFS= read -r line || [[ -n "$line" ]]; do - trimmed="$(trim "$line")" - if [[ "$trimmed" == *=* || "$trimmed" == *:* ]]; then - name="$(trim "${trimmed%%[=:]*}")" - value="$(trim "${trimmed#"$name"}")" - value="$(trim "${value#[:=]}")" - if [[ "$name" =~ ^[A-Za-z_][A-Za-z0-9_]*_(FILE|SOURCE)$ ]]; then - value="$(trim "${value%%#*}")" - value="${value#\"}"; value="${value%\"}" - value="${value#\'}"; value="${value%\'}" - if [[ -n "$value" ]] && ! is_safe_absolute_path "$value"; then - echo "unsafe path value for $name in $source" >&2 - return 1 - fi - fi - fi - done <"$source" -} - -require_absent() { - local source="$1" label="$2" - shift 2 - local forbidden - for forbidden in "$@"; do - if grep -Fq -- "$forbidden" "$source"; then - echo "$label contains forbidden text: $forbidden" >&2 - return 1 - fi - done -} - -require_pattern() { - local source="$1" label="$2" pattern="$3" - python3 - "$source" "$label" "$pattern" <<'PY' -import pathlib, re, sys -source = pathlib.Path(sys.argv[1]).read_text() -label = sys.argv[2] -pattern = sys.argv[3] -if not re.search(pattern, source, re.MULTILINE | re.DOTALL): - raise SystemExit(f"{label} lacks required pattern: {pattern}") -PY -} - -require_headings() { - local source="$1" label="$2" - shift 2 - local heading - for heading in "$@"; do - grep -Fqx "## $heading" "$source" || { - echo "missing required heading in $label: $heading" >&2 - return 1 - } - done +require_file() { + [[ -f "$root/$1" ]] || fail "missing required file: $1" } require_text() { - local source="$1" label="$2" - shift 2 - local expected - for expected in "$@"; do - grep -Fq -- "$expected" "$source" || { - echo "$label lacks required instruction: $expected" >&2 - return 1 - } - done + local path="$1" text="$2" + grep -Fq -- "$text" "$root/$path" || fail "$path must contain: $text" } -require_concept_tokens() { - local source="$1" label="$2" - shift 2 - python3 - "$source" "$label" "$@" <<'PY' -import pathlib, re, sys -text = pathlib.Path(sys.argv[1]).read_text().lower() -label = sys.argv[2] -tokens = [t.lower() for t in sys.argv[3:]] -for token in tokens: - if token not in text: - raise SystemExit(f"{label} lacks required concept token: {token}") -PY -} - -verify_workspace_descriptor_doc_contract() { - local source="$1" label="$2" - if ! PYTHONDONTWRITEBYTECODE=1 python3 "$root/scripts/workspace_descriptor_doc_contract.py" --document "$source"; then - echo "$label violates the workspace descriptor documentation contract" >&2 - return 1 +forbid_text() { + local path="$1" text="$2" + if grep -Fq -- "$text" "$root/$path"; then + fail "$path retains obsolete reference: $text" fi } -verify_markdown_table_relationships() { - local source="$1" label="$2" heading="$3" spec_json="$4" - python3 - "$source" "$label" "$heading" "$spec_json" <<'PY' -import json, pathlib, re, sys -path = pathlib.Path(sys.argv[1]) -label = sys.argv[2] -heading = sys.argv[3] -spec = json.loads(sys.argv[4]) -text = path.read_text() -match = re.search(rf"^##+\s+{re.escape(heading)}\s*$", text, re.MULTILINE) -if not match: - raise SystemExit(f"{label}: missing structured section '{heading}'") -lines = text[match.end():].splitlines() -table = [] -for line in lines: - if not line.strip(): - if table: - break - continue - if not line.lstrip().startswith("|"): - if table: - break - continue - table.append(line.rstrip()) -if len(table) < 3: - raise SystemExit(f"{label}: structured table '{heading}' is incomplete") -headers = [cell.strip().lower() for cell in table[0].strip().strip("|").split("|")] -rows = [] -for raw in table[2:]: - cells = [cell.strip() for cell in raw.strip().strip("|").split("|")] - if len(cells) != len(headers): - raise SystemExit(f"{label}: malformed row in '{heading}'") - rows.append(dict(zip(headers, cells))) -for row_spec in spec["rows"]: - found = False - for row in rows: - ok = True - for column, pattern in row_spec.items(): - value = row.get(column.lower(), "") - if not re.search(pattern, value, re.IGNORECASE | re.DOTALL): - ok = False - break - if ok: - found = True - break - if not found: - raise SystemExit(f"{label}: missing relationship in '{heading}': {row_spec}") -PY -} - -semantic_index_relationship_spec() { - cat <<'JSON' -{"rows":[ - {"scope":"workspace semantic index","ownership rule":"(each|one|single|exactly one).*(workspace).*(single|one|exactly one).*(Qdrant).*(collection)|(workspace keeps exactly one qdrant collection reserved for itself)","isolation rule":"schema.*evidence.*memory.*(one|that).*(collection).*(kind|payload)|schema.*evidence.*memory.*together.*collection.*(kind|payload)"} -]} -JSON -} - -verify_compose_internal_semantic_contract() { +verify_compose_topology() { python3 - "$root/compose.yaml" <<'PY' -import sys, yaml, pathlib -doc = yaml.safe_load(pathlib.Path(sys.argv[1]).read_text()) -services = doc["services"] -expected = {"core", "frontend", "qdrant", "embedding", "embedding-model-init", "workspace-maintenance"} +import pathlib +import sys +import yaml + +compose = yaml.safe_load(pathlib.Path(sys.argv[1]).read_text()) +expected = { + "catalog-db", "catalog-migrate", "core", "embedding", "embedding-model-init", + "frontend", "qdrant", "workspace-maintenance", +} +services = compose.get("services", {}) if set(services) != expected: raise SystemExit(f"compose.yaml services mismatch: {sorted(services)}") -if doc["services"]["workspace-maintenance"].get("profiles") != ["workspace-maintenance"]: - raise SystemExit("workspace-maintenance must be profile-gated and absent from default startup") -core = services["core"] -env = core["environment"] -for key, value in { - "THT_INTERNAL_QDRANT_URL": "http://qdrant:6333", - "THT_INTERNAL_EMBEDDING_URL": "http://embedding:11434", - "THT_INTERNAL_EMBEDDING_MODEL": "qwen3-embedding:0.6b", - "THT_INTERNAL_EMBEDDING_DIMENSIONS": "1024", -}.items(): - if env.get(key) != value: - raise SystemExit(f"core missing semantic env {key}={value}") -for forbidden in ("THT_VEC_REST_URL", "THT_VEC_WRITE_REST_URL", "THT_OLLAMA_URL"): - if forbidden in env: - raise SystemExit(f"core still exposes deprecated env {forbidden}") -if core["depends_on"]["qdrant"]["condition"] != "service_healthy": - raise SystemExit("core must wait for qdrant health") -if core["depends_on"]["embedding-model-init"]["condition"] != "service_completed_successfully": - raise SystemExit("core must wait for model init success") -for name, port in (("qdrant", "6333"), ("embedding", "11434")): - service = services[name] - if "ports" in service: - raise SystemExit(f"{name} must stay private") - if service.get("expose") != [port]: - raise SystemExit(f"{name} expose mismatch") -if "devices" in str(services["embedding"]): - raise SystemExit("base embedding service must stay CPU-first") -volumes = set(doc["volumes"]) -for required in ("qdrant-data", "embedding-models", "workspace-secrets"): - if required not in volumes: - raise SystemExit(f"missing volume {required}") -model_init = services["embedding-model-init"] -if model_init["environment"].get("OLLAMA_MODEL") != "qwen3-embedding:0.6b": - raise SystemExit("model init must pin qwen3-embedding:0.6b") +if services["catalog-migrate"].get("profiles") != ["catalog-maintenance"]: + raise SystemExit("catalog-migrate must remain an explicit catalog-maintenance operation") +if services["workspace-maintenance"].get("profiles") != ["workspace-maintenance"]: + raise SystemExit("workspace-maintenance must remain profile-gated") +if services["catalog-migrate"].get("depends_on", {}).get("catalog-db", {}).get("condition") != "service_healthy": + raise SystemExit("catalog-migrate must wait for healthy catalog-db") PY + require_text scripts/run-stack.sh 'up -d catalog-db' + require_text scripts/run-stack.sh 'run --rm catalog-migrate' + echo "Compose topology contract passed" } -verify_workspace_descriptor_semantic_contract() { - local source="${1:?source required}" - local label="${2:-$source}" - python3 - "$source" "$label" <<'PY' -import pathlib, sys, yaml -path = pathlib.Path(sys.argv[1]) -label = sys.argv[2] -doc = yaml.safe_load(path.read_text()) -ws = doc["workspace"] -semantic = doc["semantic_index"] -vector = semantic["vector_store"] -embedding = semantic["embedding"] -if ws["schema_version"] != 3: - raise SystemExit(f"{label}: schema_version must be 3") -if vector["engine"] != "qdrant": - raise SystemExit(f"{label}: vector store must be qdrant") -if vector["collection"] != ws["id"]: - raise SystemExit(f"{label}: collection must equal workspace id") -if vector["dimensions"] != 1024 or vector["distance"] != "cosine": - raise SystemExit(f"{label}: vector contract must be 1024/cosine") -if embedding["provider"] != "ollama_internal": - raise SystemExit(f"{label}: embedding provider must be ollama_internal") -if embedding["model"] != "qwen3-embedding:0.6b": - raise SystemExit(f"{label}: embedding model must be qwen3-embedding:0.6b") -if embedding["dimensions"] != 1024: - raise SystemExit(f"{label}: embedding dimensions must be 1024") -PY -} - - -verify_workspace_evidence_contract() { - local base_root="${1:-$root}" - python3 - "$base_root" <<'PY' -import pathlib, re, sys, yaml -from pathlib import PurePosixPath - -base = pathlib.Path(sys.argv[1]) -contract_path = base / "docs/contracts/workspace-evidence-v3.md" -local_path = base / "docs/install/local-workspace-registry.md" -server_path = base / "docs/install/server-workspace-registry.md" -readme_path = base / "README.md" -migration_path = base / "docs/migrations/p1-to-p1-1-registry-layout.md" -bindings_path = base / "docs/install/examples/workspace-bindings.env.example" -descriptor_paths = [ - base / "deploy/workspaces/example.yaml", - base / "deploy/workspaces/psd.yaml.example", -] -paths = [contract_path, local_path, server_path, readme_path, migration_path, bindings_path, *descriptor_paths] -for path in paths: - if not path.is_file(): - raise SystemExit(f"missing workspace Evidence contract input: {path.relative_to(base)}") - -for path in descriptor_paths: - relative = path.relative_to(base).as_posix() - document = yaml.safe_load(path.read_text()) - workspace_id = document["workspace"]["id"] - evidence = document.get("evidence") - if not isinstance(evidence, dict) or not isinstance(evidence.get("source"), dict): - raise SystemExit(f"{relative}: missing explicit filesystem Evidence contract") - uri = evidence["source"].get("uri") - expected_uri = f"{workspace_id}/evidence" - if not isinstance(uri, str) or uri.startswith("/") or "\\" in uri or ".." in uri.split("/"): - raise SystemExit(f"{relative}: noncanonical filesystem Evidence URI") - if uri != expected_uri: - raise SystemExit(f"{relative}: Evidence namespace mismatch") - expected = { - "source": { - "type": "filesystem", - "uri": expected_uri, - "patterns": ["**/*.md"], - "max_bytes": 10485760, - }, - "policy": {"max_chunk_chars": 4000, "retain_published_generations": 3}, - } - if evidence != expected: - raise SystemExit(f"{relative}: explicit filesystem Evidence object mismatch") - -contract = contract_path.read_text() -readme = readme_path.read_text() -migration = migration_path.read_text() -all_public = "\n".join(path.read_text() for path in paths) -active_public = "\n".join(path.read_text() for path in [contract_path, local_path, server_path, readme_path, bindings_path, *descriptor_paths]) - - -def normalize_space(text: str) -> str: - return re.sub(r"\s+", " ", text.strip()) - - -def named_example(name): - match = re.search( - rf"^### Example: {re.escape(name)}\s*$\n\s*```yaml\n(.*?)^```\s*$", - contract, - re.MULTILINE | re.DOTALL, - ) - if not match: - raise SystemExit(f"missing named {name} Evidence YAML example") - return yaml.safe_load(match.group(1)) - -examples = { - "filesystem": { - "evidence": { - "schema_version": 2, - "source": { - "type": "filesystem", "uri": "example/evidence", - "patterns": ["curated/**/*.md"], "max_bytes": 10485760, - }, - "policy": {"max_chunk_chars": 4000, "retain_published_generations": 3}, - }, - }, - "http": { - "evidence": { - "source": { - "type": "http", "uris": ["https://evidence.example.invalid/report.md"], - "authentication": "signed_urls_file", "connect_timeout_ms": 5000, - "read_timeout_ms": 30000, "max_bytes": 10485760, "max_redirects": 5, - "allow_private_hosts": False, "max_cache_bytes": 67108864, - }, - "policy": {"max_chunk_chars": 4000, "retain_published_generations": 3}, - }, - }, - "s3": { - "evidence": { - "source": { - "type": "s3", "uri": "s3://example-evidence/curated/", - "credentials": "static_files", "trusted_endpoint": False, - "allow_private_endpoint": False, "allow_insecure_endpoint": False, - "max_bytes": 10485760, "max_objects": 10000, "max_pages": 100, - "page_size": 1000, - }, - "policy": {"max_chunk_chars": 4000, "retain_published_generations": 3}, - }, - }, -} -for name, expected in examples.items(): - if named_example(name) != expected: - raise SystemExit(f"{name} Evidence YAML example shape/default mismatch") - -required_contract_phrases = [ - "Evidence is optional: a valid v3 descriptor without it remains operational.", - "reject unknown keys", - "nonempty list of unique, normalized relative POSIX globs", - "no whitespace, control character, backslash, userinfo, query, or fragment", - "A custom endpoint requires", - "HTTP endpoint additionally requires", - "page size cannot exceed 1000", - "Public docs, APIs, and rendered YAML never expose file contents.", - "THT_WORKSPACE_SECRET_ROOTS", - "readable regular file", - "strictly below", - "Content-only revision", - "`schema_version` value `1`", - "It is authoritative for workspace ID,\nname, description, and display order.", - "The descriptor at `/workspace.yaml` must match the\ncatalog metadata exactly.", - "catalog-only entries are invalid and reject the complete candidate revision.", - "The API and runtime never write\n`thoth-workspaces.yaml`, `/workspace.yaml`, `/schema/**`, or `/evidence/**` in the\nauthoring repository.", -] -normalized_contract = normalize_space(contract) -for phrase in required_contract_phrases: - if normalize_space(phrase) not in normalized_contract: - raise SystemExit(f"workspace Evidence contract lacks required rule: {phrase}") - -mode_rules = { - "missing public HTTP mode": "Public HTTP (`authentication: none`) uses the declared query-free\nURIs directly and requires no Evidence credential file.", - "missing ambient S3 mode": "Ambient S3\n(`credentials: ambient`) uses the runtime provider chain and requires no Evidence credential file.", -} -for error, phrase in mode_rules.items(): - if phrase not in contract: - raise SystemExit(error) -if "positive safe integers" not in contract or "nonnegative safe integer" not in contract or "9007199254740991" not in contract: - raise SystemExit("missing strict Evidence numeric domains") -if "Endpoint-policy flags cannot be enabled without `endpoint_url`." not in contract: - raise SystemExit("missing S3 endpoint policy without endpoint invariant") - -for forbidden in ( - "workspace-content//evidence", - "workspaces/.yaml", - "workspace-content/example/evidence", - "Validate and publish the descriptor against that base commit", -): - if forbidden in active_public: - raise SystemExit("old registry layout text found") - -required_tree_lines = [ - "workspace-repository.git/", "├── thoth-workspaces.yaml", "├── example/", - "│ ├── workspace.yaml", "│ └── evidence/...", "└── another/", - " └── workspace.yaml", -] -if any(line not in contract for line in required_tree_lines): - raise SystemExit("missing canonical Evidence layout") - - -def table_for(heading): - match = re.search(rf"^## {re.escape(heading)}\s*$", contract, re.MULTILINE) - if not match: - raise SystemExit(f"missing structured Evidence section: {heading}") - rows = [] - for line in contract[match.end():].splitlines(): - if line.startswith("## "): - break - if line.startswith("|"): - cells = [cell.strip() for cell in line.strip().strip("|").split("|")] - if len(cells) >= 2 and not all(set(cell) <= {"-", ":"} for cell in cells): - rows.append(cells) - return rows[1:] if rows else [] - -relationships = {row[0]: row[1] for row in table_for("Registry revision and phase ownership")} -revision_text = relationships.get("Revision identity", "") -if not all(token in revision_text for token in ("same 40-hex Git commit", "catalog blob", "descriptor blob", "root tree")): - raise SystemExit("missing same-revision ownership") -if "Evidence-only commit" not in relationships.get("Content-only revision", "") or "revision.commit" not in relationships.get("Content-only revision", ""): - raise SystemExit("missing content-only revision identity") -repository_consumer = relationships.get("Repository consumer", "") -if not all(token in repository_consumer for token in ("complete candidate", "atomically activates", "never edits, commits, or pushes")): - raise SystemExit("missing read-only repository-consumer rule") -runtime_secrets = relationships.get("Runtime secrets", "") -if not all(token in runtime_secrets for token in ("configured/missing status only", "runtime lease")): - raise SystemExit("missing runtime-secret lifecycle rule") -p11 = relationships.get("P1.1", "") -p6 = relationships.get("P6", "") -if not all(token in p11 for token in ("lexical URI `/evidence`", "Git tree", "same commit", "does not recursively inspect nested symlinks", "out of scope for P1.1")): - raise SystemExit("missing P1.1 lexical/tree ownership") -if not all(token in p6 for token in ("commit-addressed materialization", "realpath", "recursive containment", "nested-symlink", "race")): - raise SystemExit("missing P6 materialization ownership") -no_scope = "P1.1 performs no acquisition, extraction, preprocessing/indexing, embeddings, Qdrant writes, active-snapshot retention, or GC." -p1_adverbs = r"(?:\s+(?:also|then|now|directly|itself))*" -p1_base_operation = r"""(?: - acquire|materialize|extract|preprocess|index|retain| - (?:create|generate)\s+embeddings?| - write\s+(?:embeddings?\s+)?to\s+Qdrant| - publish\s+`?ACTIVE\b`?| - garbage[- ]collect| - (?:run|perform)\s+(?:retention|GC|garbage[ -]collection) -)""" -p1_third_person_operation = r"""(?: - acquires|materializes|extracts|preprocesses|indexes|retains| - (?:creates|generates)\s+embeddings?| - writes\s+(?:embeddings?\s+)?to\s+Qdrant| - publishes\s+`?ACTIVE\b`?| - garbage[- ]collects| - (?:runs|performs)\s+(?:retention|GC|garbage[ -]collection) -)""" -p1_ownership = r"""(?: - (?:owns|handles|performs)|is\s+responsible\s+for -)\s+(?:Evidence\s+)?(?: - acquisition|materialization|extraction|preprocessing|embeddings?| - Qdrant\s+writes?|indexing|`?ACTIVE`?\s+publication|retention|GC| - garbage[ -]collection -)""" -positive_p1_operation = re.compile( - rf"""\bP1(?:\.1)?\b{p1_adverbs}\s+(?: - (?:(?:can|may|must|will|should|does){p1_adverbs}\s+){p1_base_operation}| - {p1_third_person_operation}| - {p1_ownership} - )\b""", - re.IGNORECASE | re.VERBOSE, -) -if normalize_space(no_scope) not in normalized_contract or positive_p1_operation.search(contract): - raise SystemExit("P1.1 scope violation") - -installation_rows = {row[0]: row[1:] for row in table_for("Installation files")} -http_row = " ".join(installation_rows.get("Signed HTTP", [])) -if "THT_WS__EVIDENCE_SIGNED_URLS_FILE" not in http_row or not all( - token in http_row for token in ( - "1048576 bytes", "nonempty UTF-8 JSON string array", "declared-URI order", - "query-stripped identities", "one-to-one", - ) -): - raise SystemExit("missing signed HTTP file boundary") -s3_pair = " ".join(installation_rows.get("Static S3 pair", [])) -if not all(token in s3_pair for token in ( - "THT_WS__EVIDENCE_ACCESS_KEY_FILE", - "THT_WS__EVIDENCE_SECRET_KEY_FILE", "Required together", "65536 bytes", -)): - raise SystemExit("missing static S3 file boundary") -s3_token = " ".join(installation_rows.get("Static S3 session", [])) -if not all(token in s3_token for token in ( - "THT_WS__EVIDENCE_SESSION_TOKEN_FILE", "Optional", "65536 bytes", -)): - raise SystemExit("missing static S3 session-token boundary") - -if "tht config check -c " not in contract: - raise SystemExit("exact config-check ordering missing") -automated = re.findall(r"^automated integration: (?:PENDING|PASS|FAIL)$", contract, re.MULTILINE) -manual = re.findall(r"^manual acceptance: (?:PENDING|PASS|FAIL)$", contract, re.MULTILINE) -if len(automated) != 1 or len(manual) != 1: - raise SystemExit("separate automated/manual states missing") - -flow_tokens = [ - "Create a local workspace", - "thoth-workspaces.yaml", - "/workspace.yaml", - "commit", - "push", - "ThothII", - "Update workspace repository", - "workspace-secrets", - "Validate workspace source", - "Test workspace connections", - "Save entered secrets", - "Forget stored value", -] -for guide in (local_path, server_path): - text = guide.read_text() - match = re.search( - r"^## Prepare and publish a workspace source\s*$\n(.*?)(?=^## |\Z)", - text, - re.MULTILINE | re.DOTALL, - ) - if not match: - raise SystemExit(f"{guide.name}: missing workspace source flow") - section = text - positions = [section.find(token) for token in flow_tokens] - if any(position < 0 for position in positions): - raise SystemExit(f"{guide.name}: curator flow missing registry rule") - -readme_required = [ - "thoth-workspaces.yaml", - "/workspace.yaml", - "/evidence/**", - "authoritative for workspace ID, name, description, and\ndisplay order", - "the complete candidate is rejected", - "ThothII\nnever writes any workspace repository content.", - "docs/migrations/p1-to-p1-1-registry-layout.md", -] -normalized_readme = normalize_space(readme) -for phrase in readme_required: - if normalize_space(phrase) not in normalized_readme: - raise SystemExit("README registry overview incomplete") - -migration_commit_phrases = [ - "git mv workspaces/.yaml /workspace.yaml", - "git mv workspace-content//evidence /evidence", - "create and review thoth-workspaces.yaml from descriptor metadata", -] -for phrase in migration_commit_phrases: - if phrase not in migration: - raise SystemExit("migration guide missing commit step") -if "Upgrade ThothII only after that migration commit is pushed." not in migration: - raise SystemExit("migration guide missing upgrade ordering") -if "Roll back the application revision and registry commit together." not in migration: - raise SystemExit("migration guide missing rollback rule") -if "reject the old flat layout and a\nrepository without `thoth-workspaces.yaml`" not in migration: - raise SystemExit("migration guide missing rejection rule") - -aws_access_key = re.compile( - r"(?") -for name, value in assignments: - lowered = value.lower() - if any(token in lowered for token in unsafe_placeholders): - raise SystemExit("unsafe file placeholder/path") - if name.endswith(("_FILE", "_SOURCE")) and value and not safe_absolute(value): - raise SystemExit("unsafe file placeholder/path") - if "_EVIDENCE_" in name and name.endswith("_FILE") and not value.startswith("/run/secrets/"): - raise SystemExit("unsafe file placeholder/path") - if "_EVIDENCE_" in name and name.endswith("_SOURCE") and not ( - value.startswith("/srv/thothii/secrets/") - or value.startswith("/absolute/path/installation-secrets/") - ): - raise SystemExit("unsafe file placeholder/path") - credential_name = re.search(r"(?:SECRET_KEY|ACCESS_KEY|PASSWORD|SESSION_TOKEN|SIGNED_URLS|CREDENTIAL)$", name) - if credential_name and value: - raise SystemExit("credential literal forbidden") - if re.match(r"(?i)(?:AKIA|ASIA)[A-Z0-9]{12,}", value): - raise SystemExit("credential literal forbidden") - if re.match(r"https?://", value) and "?" in value: - raise SystemExit("query-bearing public URI forbidden") - -for uri in re.findall(r"https?://[^\s`\"'<>]+", all_public): - if "?" in uri: - raise SystemExit("query-bearing public URI forbidden") - authority = uri.split("//", 1)[1].split("/", 1)[0] - if "@" in authority: - raise SystemExit("credential literal forbidden") - -expected_evidence_bindings = { - "THT_WS_SIGNED_HTTP_EVIDENCE_SIGNED_URLS_FILE": "/run/secrets/signed-http-evidence-urls.json", - "THT_WS_STATIC_S3_EVIDENCE_ACCESS_KEY_FILE": "/run/secrets/static-s3-evidence-access-key", - "THT_WS_STATIC_S3_EVIDENCE_SECRET_KEY_FILE": "/run/secrets/static-s3-evidence-secret-key", - "THT_WS_STATIC_S3_EVIDENCE_SESSION_TOKEN_FILE": "/run/secrets/static-s3-evidence-session-token", -} -binding_values = dict(dotenv_lines(bindings_path)) -for name, value in expected_evidence_bindings.items(): - if binding_values.get(name) != value: - raise SystemExit(f"workspace bindings example mismatch: {name}") - -print("workspace Evidence documentation contract passed") -PY -} - -verify_vector_helper_interfaces() { - local output status - output="$(mktemp "${tmp_prefix}thoth-vector-backup-help.XXXXXX")" - set +e - "$root/scripts/vector-backup.sh" >"$output" 2>&1 - status=$? - set -e - [[ $status -eq 2 ]] || { cat "$output" >&2; rm -f "$output"; echo "vector-backup usage exit mismatch" >&2; return 1; } - grep -Eq 'usage: .*--project-name NAME --output FILE' "$output" || { cat "$output" >&2; rm -f "$output"; echo "vector-backup usage contract changed" >&2; return 1; } - set +e - "$root/scripts/vector-restore.sh" >"$output" 2>&1 - status=$? - set -e - [[ $status -eq 2 ]] || { cat "$output" >&2; rm -f "$output"; echo "vector-restore usage exit mismatch" >&2; return 1; } - grep -Eq 'usage: .*--project-name NAME --input FILE --confirm-project NAME' "$output" || { cat "$output" >&2; rm -f "$output"; echo "vector-restore usage contract changed" >&2; return 1; } - rm -f "$output" -} - -verify_project_state_current_contract() { - local source="${1:-$root/PROJECT_STATE.md}" - local label="${2:-PROJECT_STATE.md}" - if ! PYTHONDONTWRITEBYTECODE=1 python3 "$root/scripts/workspace_descriptor_doc_contract.py" --project-state "$source"; then - echo "$label violates the workspace descriptor documentation contract" >&2 - return 1 - fi - python3 - "$source" "$label" <<'PY' -import pathlib, re, sys -text = pathlib.Path(sys.argv[1]).read_text() -label = sys.argv[2] -marker = re.search(r"^# Historical archive$", text, re.MULTILINE) -if not marker: - raise SystemExit(f"{label}: missing Historical archive boundary") -current = text[:marker.start()] -if not re.search(r"Internal Qdrant \+ Ollama semantic infrastructure", current, re.MULTILINE): - raise SystemExit(f"{label}: current section missing internal semantic snapshot heading") -if not re.search(r"\b(one|single)\b.*\bworkspace\b.*\b(one|single)\b.*\bQdrant\b.*\bcollection\b", current, re.IGNORECASE | re.DOTALL): - raise SystemExit(f"{label}: current section must describe one-workspace/one-collection ownership") -if not re.search(r"\bDWH\b", current) or not re.search(r"\bLLM\b", current): - raise SystemExit(f"{label}: current section must identify DWH and LLM") -if not re.search(r"\bexternal\b", current, re.IGNORECASE): - raise SystemExit(f"{label}: current section must mark the external boundary") -if "embedding-model-init" not in current: - raise SystemExit(f"{label}: current section missing embedding-model-init") -forbidden = [ - r"supported Compose stack is exactly `frontend` plus `core`", - r"DWH, vector DB, embedding, LLM", - r"vector DB, embedding, and LLM remain external", -] -for pattern in forbidden: - if re.search(pattern, current, re.MULTILINE): - raise SystemExit(f"{label}: current section still contains contradictory active text: {pattern}") -PY -} - -verify_internal_semantic_infrastructure_docs() { - local readme="$root/README.md" - local agents="$root/AGENTS.md" - local local_manual="$root/docs/install/local-workspace-registry.md" - local server_manual="$root/docs/install/server-workspace-registry.md" - local compact_manual="$root/docs/installazione-docker-4-contesti.md" - local diagnostics="$root/docs/workspace-diagnostic-protocol.md" - local memory="$root/docs/gestione-memory.md" - local secrets="$root/deploy/secrets/README.md" - - verify_compose_internal_semantic_contract || return 1 - verify_workspace_descriptor_semantic_contract "$root/deploy/workspaces/example.yaml" "example workspace" || return 1 - verify_workspace_descriptor_semantic_contract "$root/deploy/workspaces/psd.yaml.example" "psd workspace example" || return 1 - verify_vector_helper_interfaces || return 1 - verify_project_state_current_contract "$root/PROJECT_STATE.md" "PROJECT_STATE.md" || return 1 - verify_workspace_descriptor_doc_contract "$readme" "README" || return 1 - verify_workspace_descriptor_doc_contract "$local_manual" "local workspace manual" || return 1 - verify_workspace_descriptor_doc_contract "$server_manual" "server workspace manual" || return 1 - verify_workspace_descriptor_doc_contract "$diagnostics" "workspace diagnostic protocol" || return 1 - - local ownership_spec semantic_index_spec compact_spec - ownership_spec='{"rows":[ - {"component":"^DWH$","ownership":"^External$","operator contract":"external|endpoint|installation"}, - {"component":"^LLM$","ownership":"^External$","operator contract":"external|endpoint|policy"}, - {"component":"^Qdrant$","ownership":"^Internal$","operator contract":"internal|Compose|qdrant-data"}, - {"component":"^Ollama embedding$","ownership":"^Internal$","operator contract":"internal|Compose|qwen3-embedding:0\\.6b"} - ]}' - semantic_index_spec="$(semantic_index_relationship_spec)" - compact_spec='{"rows":[ - {"componente":"^DWH$","ownership":"^Esterno$","contratto operativo":"endpoint.*estern"}, - {"componente":"^LLM$","ownership":"^Esterno$","contratto operativo":"esterna|esterno"}, - {"componente":"^Qdrant$","ownership":"^Interno$","contratto operativo":"interno.*obbligatorio.*qdrant-data"}, - {"componente":"^Ollama embedding$","ownership":"^Interno$","contratto operativo":"interno.*obbligatorio.*qwen3-embedding:0\\.6b"} - ]}' - - verify_markdown_table_relationships "$local_manual" "local workspace manual" "Architecture ownership contract" "$ownership_spec" || return 1 - verify_markdown_table_relationships "$server_manual" "server workspace manual" "Architecture ownership contract" "$ownership_spec" || return 1 - verify_markdown_table_relationships "$local_manual" "local workspace manual" "Semantic index ownership contract" "$semantic_index_spec" || return 1 - verify_markdown_table_relationships "$server_manual" "server workspace manual" "Semantic index ownership contract" "$semantic_index_spec" || return 1 - verify_markdown_table_relationships "$compact_manual" "four-context install note" "Contratto sintetico di ownership" "$compact_spec" || return 1 - - require_pattern "$readme" "README" 'mandatory stack.+qdrant.+embedding.+embedding-model-init' || return 1 - require_pattern "$readme" "README" 'qwen3-embedding:0\.6b' || return 1 - require_pattern "$readme" "README" 'qdrant-data.+embedding-models' || return 1 - require_pattern "$readme" "README" 'confirm-project' || return 1 - require_pattern "$agents" "AGENTS.md" 'Qdrant and Ollama are internal Compose services' || return 1 - require_pattern "$agents" "AGENTS.md" 'DWH and LLM remain external configuration endpoints' || return 1 - for manual in "$local_manual" "$server_manual"; do - require_pattern "$manual" "$(basename "$manual")" 'qwen3-embedding:0\.6b' || return 1 +verify_navigation() { + local path + for path in \ + install/first-start.md \ + operations/workspaces.md \ + operations/database-management.md \ + guida-utente.md \ + architecture/overview.md \ + contracts/catalog-schema-snapshot.md; do + require_file "docs/$path" + require_text mkdocs.yml "$path" done - require_pattern "$local_manual" "local workspace manual" 'CPU-first' || return 1 - require_pattern "$local_manual" "local workspace manual" 'THOTH_ENABLE_EMBEDDING_GPU=1' || return 1 - require_pattern "$server_manual" "server workspace manual" 'Qdrant backup/restore' || return 1 - require_pattern "$compact_manual" "four-context install note" '1024 dimensioni' || return 1 - require_pattern "$diagnostics" "workspace diagnostic protocol" 'semantic_index_incompatible' || return 1 - require_absent "$diagnostics" "workspace diagnostic protocol" \ - 'engine: pgvector' \ - 'provider: ollama_compatible' \ - 'THT_WS__VECTOR_TRANSPORT' \ - 'THT_WS__EMBEDDING_BASE_URL' || return 1 - require_pattern "$memory" "memory guide" 'Indice Qdrant' || return 1 - require_pattern "$memory" "memory guide" 'indice derivato ma persistente' || return 1 - require_pattern "$memory" "memory guide" '`kind`' || return 1 - require_absent "$memory" "memory guide" \ - 'Indice pgvector' \ - "all'indice pgvector" || return 1 - require_pattern "$secrets" "deploy secrets guide" 'THT_MODEL_API_KEY.+THT_DWH_API_KEY.+THT_CA.+THT_SSL_CA' || return 1 - require_pattern "$secrets" "deploy secrets guide" 'Do not add vector or embedding endpoint credentials to the bundle' || return 1 - require_absent "$secrets" "deploy secrets guide" 'PI_PROVIDER_API_KEY' || return 1 + echo "Current documentation navigation contract passed" } -verify_local_guide() { - local guide="$root/docs/install/local.md" - [[ -f "$guide" ]] || { - echo "missing local installation guide: docs/install/local.md" >&2 - return 1 - } - require_headings "$guide" "local installation guide" \ - "Choose your platform" \ - "Prerequisites" \ - "Clone and verify LF" \ - "Create the local operator files" \ - "Address external services" \ - "Build ThothII and tht" \ - "Start and verify" \ - "Update an installation" \ - "Back up and restore" \ - "Data-preserving uninstall" \ - "Next: workspaces and Pi" - require_text "$guide" "local installation guide" \ - "git clone" \ - "bash scripts/verify-line-endings.sh" \ - "deploy/env/local.env" \ - "host.docker.internal" \ - "host-gateway" \ - "container 127.0.0.1" \ - "bash scripts/build-local.sh" \ - "scripts/build-local.ps1" \ - "bash scripts/build-tht.sh" \ - "tht --installation" \ - "curl --fail http://127.0.0.1:8080/health" \ - "http://127.0.0.1:8080" \ - "git pull --ff-only" \ - "docker compose down --volumes" - node - "$guide" <<'NODE' -const fs = require("fs"); -const source = fs.readFileSync(process.argv[2], "utf8"); +verify_install_and_workspace_guides() { + local install='docs/install/first-start.md' + local workspace='docs/operations/workspaces.md' + require_file "$install" + require_file "$workspace" -function section(name) { - const marker = `## ${name}`; - const start = source.indexOf(marker); - if (start < 0) throw new Error(`missing section: ${name}`); - const next = source.indexOf("\n## ", start + marker.length); - return source.slice(start, next < 0 ? source.length : next); -} - -function blocks(name, language) { - const expression = new RegExp("```" + language + "\\n([\\s\\S]*?)```", "g"); - return [...section(name).matchAll(expression)].map((match) => match[1]); -} - -function requireTokens(label, text, tokens) { - for (const token of tokens) { - if (!text.includes(token)) throw new Error(`${label} lacks structural token: ${token}`); - } -} - -function requirePattern(label, text, pattern) { - if (!pattern.test(text)) throw new Error(label); -} - -let inCodeFence = false; -for (const line of source.split(/\n/)) { - if (line.trimStart().startsWith("```")) { - inCodeFence = !inCodeFence; - continue; - } - if (!line.includes("docker compose down --volumes")) continue; - const normalized = line.toLowerCase().replaceAll("*", ""); - if (inCodeFence || !/(do not|never)/.test(normalized) || /^\s*(docker|&?\s*docker)/.test(normalized)) { - throw new Error("docker compose down --volumes must appear only in an explicit prose prohibition"); - } -} - -const setupPowerShell = blocks("Create the local operator files", "powershell").join("\n"); -requireTokens("native PowerShell setup", setupPowerShell, [ - "Copy-Item", "New-Item", "icacls.exe", "/inheritance:r", "/grant:r", - "WindowsIdentity", "deploy/env/local.env.example", "thothii-installation.yaml", -]); - -const healthPowerShell = blocks("Start and verify", "powershell").join("\n"); -requireTokens("native PowerShell health", healthPowerShell, [ - "curl.exe --fail", "http://127.0.0.1:8080/health", "http://127.0.0.1:8787/health", - "pi doctor", "pi test", -]); - -const updateShell = blocks("Update an installation", "sh").join("\n"); -requireTokens("installation-aware source update", updateShell, [ - "NEXT_PI_VERSION", "RUNNING_PI_VERSION", "--source build", "git rev-parse HEAD", - "pi status", "status", "doctor", "curl --fail", "set -euo pipefail", - "git status --porcelain --untracked-files=all", "USES_BASE_CORE", "thothii-core:local", -]); -if (/\|\|\s*true|;\s*true\b/.test(updateShell)) throw new Error("POSIX source update contains a failure-bypass command"); -requirePattern("POSIX source update does not fail closed: source pull", updateShell, - /if ! git pull --ff-only; then abort_update/); -requirePattern("POSIX source update does not fail closed: installation status", updateShell, - /if ! INSTALLATION_STATUS="\$\("\$THT_BIN" --installation "\$INSTALLATION" status\)"; then/); -requirePattern("POSIX source update does not fail closed: Pi status", updateShell, - /if ! RUNNING_PI_VERSION="\$\("\$THT_BIN" --installation "\$INSTALLATION" pi status\)"; then/); -requirePattern("POSIX source update does not fail closed: local build", updateShell, - /if ! bash scripts\/build-local\.sh; then/); -requirePattern("POSIX source update does not fail closed: tht build", updateShell, - /if ! bash scripts\/build-tht\.sh; then/); -requirePattern("POSIX source update lacks the same-version/no-selector path", updateShell, - /if \[\[ "\$NEXT_PI_VERSION" == "\$RUNNING_PI_VERSION" \]\]; then[\s\S]*"\$USES_BASE_CORE" == true[\s\S]*TRANSACTIONAL_PI_UPDATE=false/); -for (const [label, pattern] of [ - ["installation start", /if ! "\$THT_BIN" --installation "\$INSTALLATION" start; then/], - ["frontend health", /if ! curl --fail http:\/\/127\.0\.0\.1:8080\/health; then/], - ["core health", /if ! curl --fail http:\/\/127\.0\.0\.1:8787\/health; then/], - ["final status", /if ! FINAL_STATUS="\$\("\$THT_BIN" --installation "\$INSTALLATION" status\)"; then/], - ["final Pi status", /if ! FINAL_PI_STATUS="\$\("\$THT_BIN" --installation "\$INSTALLATION" pi status\)"; then/], - ["final doctor", /if ! "\$THT_BIN" --installation "\$INSTALLATION" doctor; then/], -]) requirePattern(`POSIX source update does not fail closed: ${label}`, updateShell, pattern); -const provenance = updateShell.indexOf("printf 'Built source revision:"); -if (provenance < updateShell.lastIndexOf("require_clean_source") || - provenance < updateShell.indexOf('abort_update "final doctor failed"')) { - throw new Error("POSIX source revision provenance is printed before final checks"); -} - -const updatePowerShell = blocks("Update an installation", "powershell").join("\n"); -requireTokens("native PowerShell source update", updatePowerShell, [ - "$NextPiVersion", "$RunningPiVersion", "--source build", "git rev-parse HEAD", - "pi status", "status", "doctor", "curl.exe --fail", "throw", "$ErrorActionPreference = 'Stop'", - "git status --porcelain --untracked-files=all", "$UsesBaseCore", "thothii-core:local", - "$TransactionalPiUpdate = $false", -]); -for (const [command, step] of [ - ["git pull --ff-only", "source pull"], - ["$InstallationStatus = @(& $THT_BIN --installation $INSTALLATION status)", "installation status"], - ["$RunningPiStatus = (& $THT_BIN --installation $INSTALLATION pi status)", "Pi status"], - ["powershell -ExecutionPolicy Bypass -File scripts/build-local.ps1", "local image build"], - ["& \"C:\\Program Files\\Git\\bin\\bash.exe\" scripts/build-tht.sh", "tht build"], - ["curl.exe --fail --silent --show-error http://127.0.0.1:8080/health", "frontend health check"], - ["curl.exe --fail --silent --show-error http://127.0.0.1:8787/health", "core health check"], - ["$FinalPiStatus = (& $THT_BIN --installation $INSTALLATION pi status)", "final Pi status"], - ["& $THT_BIN --installation $INSTALLATION doctor", "final doctor"], -]) { - const commandAt = updatePowerShell.indexOf(command); - const checkAt = updatePowerShell.indexOf(`Assert-NativeSuccess '${step}'`, commandAt); - if (commandAt < 0 || checkAt < commandAt || checkAt - commandAt > 220) { - throw new Error(`PowerShell source update does not propagate failure: ${step}`); - } -} -requirePattern("PowerShell source update lacks the same-version/no-selector path", updatePowerShell, - /if \(\$NextPiVersion -eq \$RunningPiVersion\) \{[\s\S]*-not \$UsesBaseCore[\s\S]*\$TransactionalPiUpdate = \$false/); - -const backupPowerShell = blocks("Back up and restore", "powershell").join("\n"); -requireTokens("native PowerShell backup/restore", backupPowerShell, [ - "$BackupDir", "$Volume", "-czf", "$TargetVolume", "$Archive", "Split-Path -Parent", - "Split-Path -Leaf", "test -z", "-xzf", -]); - -for (const block of [setupPowerShell, healthPowerShell, updatePowerShell, backupPowerShell]) { - if (/\$\((dirname|basename)\b|\bmkdir -p\b|\bchmod\s+[0-7]/.test(block)) { - throw new Error("native PowerShell block contains a POSIX-only command sequence"); - } -} -NODE - local update_fixture update_script fake_bin calls output status - update_fixture="$(mktemp -d "${tmp_prefix}thoth-source-update.XXXXXX")" - trap 'rm -rf "$update_fixture"' RETURN - update_script="$update_fixture/update.sh" - awk ' - /^## Update an installation$/ { in_section=1; next } - in_section && /^```sh$/ { in_code=1; next } - in_code && /^```$/ { exit } - in_code { print } - ' "$guide" >"$update_script" - chmod 0700 "$update_script" - mkdir -p "$update_fixture/project/docker" "$update_fixture/project/scripts" "$update_fixture/bin" - printf 'ARG PI_VERSION=0.80.3\n' >"$update_fixture/project/docker/core.Dockerfile" - printf '%s\n' \ - '#!/bin/sh' \ - 'printf "git %s\n" "$*" >>"$CALLS"' \ - 'case "$1" in' \ - ' status) if [ "$FAIL_STEP" = dirty ]; then printf "?? untracked-build-context\n"; fi ;;' \ - ' pull) [ "$FAIL_STEP" != pull ] || exit 9 ;;' \ - ' rev-parse) printf "0123456789abcdef\n" ;;' \ - 'esac' \ - 'exit 0' >"$update_fixture/bin/git" - printf '%s\n' \ - '#!/bin/sh' \ - 'printf "bash %s\n" "$*" >>"$CALLS"' \ - 'if [ "$1" = scripts/build-local.sh ] && [ "$FAIL_STEP" = build ]; then exit 8; fi' \ - 'exit 0' >"$update_fixture/bin/bash" - printf '%s\n' \ - '#!/bin/sh' \ - 'printf "tht %s\n" "$*" >>"$CALLS"' \ - 'case " $* " in' \ - ' *" pi status "*) [ "$FAIL_STEP" != status ] || exit 7; printf "Pi version: 0.80.3\n" ;;' \ - ' *" status "*) printf "[{\"Service\":\"core\",\"Image\":\"thothii-core:local\"}]\n" ;;' \ - 'esac' \ - 'exit 0' >"$update_fixture/bin/tht" - printf '%s\n' \ - '#!/bin/sh' \ - 'printf "curl %s\n" "$*" >>"$CALLS"' \ - 'exit 0' >"$update_fixture/bin/curl" - chmod 0700 "$update_fixture/bin/git" "$update_fixture/bin/bash" \ - "$update_fixture/bin/tht" "$update_fixture/bin/curl" - - for fixture_step in clean dirty pull status build; do - calls="$update_fixture/calls-$fixture_step" - output="$update_fixture/output-$fixture_step" - : >"$calls" - set +e - ( - cd "$update_fixture/project" - env PATH="$update_fixture/bin:$PATH" CALLS="$calls" FAIL_STEP="$fixture_step" \ - THT_BIN="$update_fixture/bin/tht" INSTALLATION="$update_fixture/installation.yaml" \ - /bin/bash "$update_script" - ) >"$output" 2>&1 - status=$? - set -e - if [[ "$fixture_step" == clean ]]; then - [[ $status -eq 0 ]] || { echo "same-version/no-selector source fixture failed" >&2; return 1; } - grep -Fq 'Built source revision: 0123456789abcdef' "$output" || { - echo "successful source fixture did not report revision provenance" >&2; return 1; - } - if grep -Fq ' pi update ' "$calls"; then - echo "same-version/no-selector source fixture incorrectly invoked pi update" >&2 - return 1 - fi - grep -Fq 'bash scripts/build-local.sh' "$calls" || return 1 - grep -Fq 'tht --installation ' "$calls" || return 1 - else - [[ $status -ne 0 ]] || { echo "$fixture_step source failure fixture was accepted" >&2; return 1; } - if grep -Fq 'Built source revision:' "$output"; then - echo "$fixture_step source failure fixture claimed revision provenance" >&2 - return 1 - fi - fi + for text in \ + 'tht setup --profile local' \ + './scripts/run-stack.sh' \ + 'catalog-migrate' \ + 'tht --installation /absolute/path/thothii-installation.yaml doctor --json'; do + require_text "$install" "$text" done - echo "source update fail-closed semantics passed" - echo "local installation guide contract passed" -} - -verify_windows_line_endings_guide() { - local guide="$root/docs/install/windows-line-endings.md" - [[ -f "$guide" ]] || { - echo "missing Windows line-ending guide: docs/install/windows-line-endings.md" >&2 - return 1 - } - require_headings "$guide" "Windows line-ending guide" \ - "Recommended WSL2 clone" \ - "Repository-local LF policy" \ - "Verify after clone or pull" \ - "Recover an existing CRLF clone" - require_text "$guide" "Windows line-ending guide" \ - "git config --local core.autocrlf false" \ - "bash scripts/verify-line-endings.sh" \ - "git add --renormalize ." \ - "git checkout-index --all --force" \ - "git diff --cached --check" \ - "reclone" - node - "$guide" <<'NODE' -const fs = require("fs"); -const source = fs.readFileSync(process.argv[2], "utf8"); -const lines = source.split(/\n/); -const sectionStart = source.indexOf("## Recover an existing CRLF clone"); -const recovery = source.slice(sectionStart); -const shell = [...recovery.matchAll(/```sh\n([\s\S]*?)```/g)].map((match) => match[1]).join("\n"); -const powershell = [...recovery.matchAll(/```powershell\n([\s\S]*?)```/g)].map((match) => match[1]).join("\n"); -const commands = [ - "git add --renormalize .", - "git checkout-index --all --force --prefix=", - "bash scripts/verify-line-endings.sh", -]; -let prior = -1; -for (const command of commands) { - const index = lines.findIndex((line, candidate) => candidate > prior && line.trim().includes(command)); - if (index < 0) throw new Error(`CRLF recovery lacks ordered command: ${command}`); - prior = index; -} -for (const token of [ - "set -euo pipefail", "validate_index_export", "validate_worktree_modes", "rewrite_index_entry", "git ls-files -s -z", - "100644", "100755", "120000", "readlink", "ln -s", "if ! git checkout-index", -]) { - if (!shell.includes(token)) throw new Error(`POSIX CRLF repair lacks fail-closed semantic: ${token}`); -} -if (/\|\|\s*true|;\s*true\b/.test(shell)) throw new Error("POSIX CRLF repair contains a failure-bypass command"); -const exportAt = shell.indexOf("if ! git checkout-index"); -const validationAt = shell.indexOf("if ! validate_index_export", exportAt); -const exportedBytesAt = shell.indexOf('if ! bash scripts/verify-line-endings.sh "$REPAIR_DIR"', validationAt); -const rewriteAt = shell.indexOf("if ! git ls-files -s -z", exportedBytesAt); -const finalModesAt = shell.indexOf("if ! validate_worktree_modes; then", rewriteAt); -const finalAt = shell.indexOf("if ! bash scripts/verify-line-endings.sh; then", finalModesAt); -if ([exportAt, validationAt, exportedBytesAt, rewriteAt, finalModesAt, finalAt].some((index) => index < 0) || - !(exportAt < validationAt && validationAt < exportedBytesAt && exportedBytesAt < rewriteAt && rewriteAt < finalModesAt && finalModesAt < finalAt)) { - throw new Error("POSIX CRLF repair does not prove a complete export before destructive rewrite"); -} -for (const token of [ - "$ErrorActionPreference = 'Stop'", "Assert-NativeSuccess 'index renormalization'", - "Assert-NativeSuccess 'normalized index check'", "Assert-NativeSuccess 'index export'", - "Assert-NativeSuccess 'index inventory'", "100644", "100755", "120000", "SymbolicLink", - "-ErrorAction Stop", "$WorktreeItem", "Assert-NativeSuccess 'repaired worktree LF verification'", -]) { - if (!powershell.includes(token)) throw new Error(`PowerShell CRLF repair lacks failure propagation: ${token}`); -} -const warningPattern = /WARNING[^\n]*destructive[^\n]*(backup|commit)/i; -const powerShellWarningAt = powershell.indexOf("# WARNING: destructive copy"); -const powerShellRewriteAt = powershell.indexOf("foreach ($Entry in $IndexEntries)", powerShellWarningAt); -if (!warningPattern.test(shell.slice(Math.max(0, rewriteAt - 180), rewriteAt)) || - powerShellRewriteAt < 0 || - !warningPattern.test(powershell.slice(Math.max(0, powerShellRewriteAt - 180), powerShellRewriteAt))) { - throw new Error("worktree rewrite lacks an immediate destructive warning requiring backup/commit"); -} -NODE - local repair_root repair_script real_git partial_repo clean_repo partial_output repair_status - repair_root="$(mktemp -d "${tmp_prefix}thoth-crlf-repair.XXXXXX")" - trap 'rm -rf "$repair_root"' RETURN - repair_script="$repair_root/repair.sh" - awk ' - /^## Recover an existing CRLF clone$/ { in_section=1; next } - in_section && /^```sh$/ { in_code=1; next } - in_code && /^```$/ { exit } - in_code { print } - ' "$guide" >"$repair_script" - chmod 0700 "$repair_script" - - prepare_crlf_fixture() { - local repository="$1" - mkdir -p "$repository/scripts" - git -C "$repository" init -q - printf '*.sh text eol=lf\n' >"$repository/.gitattributes" - printf '#!/bin/sh\nexit 0\n' >"$repository/repair.sh" - printf 'target\n' >"$repository/target.txt" - cp "$root/scripts/verify-line-endings.sh" "$repository/scripts/verify-line-endings.sh" - ln -s target.txt "$repository/workspace-link" - git -C "$repository" add .gitattributes repair.sh target.txt workspace-link \ - scripts/verify-line-endings.sh 2>/dev/null - printf '#!/bin/sh\r\nexit 0\r\n' >"$repository/repair.sh" - } - - partial_repo="$repair_root/partial/worktree" - mkdir -p "$partial_repo" "$repair_root/partial/bin" - prepare_crlf_fixture "$partial_repo" - real_git="$(command -v git)" - printf '%s\n' \ - '#!/bin/sh' \ - '"$REAL_GIT" "$@"' \ - 'status=$?' \ - 'if [ $status -eq 0 ] && [ "$1" = checkout-index ]; then rm -f "$PARTIAL_EXPORT_PATH"; fi' \ - 'exit $status' >"$repair_root/partial/bin/git" - chmod 0700 "$repair_root/partial/bin/git" - partial_output="$repair_root/partial/output" - set +e - ( - cd "$partial_repo" - env PATH="$repair_root/partial/bin:$PATH" REAL_GIT="$real_git" \ - PARTIAL_EXPORT_PATH="$repair_root/partial/ThothII-lf-repair/repair.sh" \ - /bin/bash "$repair_script" - ) >"$partial_output" 2>&1 - repair_status=$? - set -e - [[ $repair_status -ne 0 ]] || { echo "partial CRLF export fixture was accepted" >&2; return 1; } - LC_ALL=C grep -q $'\r' "$partial_repo/repair.sh" || { - echo "partial CRLF export fixture rewrote bytes before complete validation" >&2; return 1; - } - [[ -L "$partial_repo/workspace-link" && "$(readlink "$partial_repo/workspace-link")" == target.txt ]] || { - echo "partial CRLF export fixture changed the tracked symlink" >&2; return 1; - } - - clean_repo="$repair_root/clean/worktree" - mkdir -p "$clean_repo" - prepare_crlf_fixture "$clean_repo" - (cd "$clean_repo" && /bin/bash "$repair_script") >/dev/null - "$root/scripts/verify-line-endings.sh" "$clean_repo" - [[ -L "$clean_repo/workspace-link" && "$(readlink "$clean_repo/workspace-link")" == target.txt ]] || { - echo "successful CRLF repair did not preserve the mode-120000 symlink" >&2; return 1; - } - echo "CRLF recovery rewrites bytes and preserves mode-120000 symlinks passed" - echo "Windows line-ending recovery guide contract passed" -} - -verify_pi_management_guide() { - local guide="$root/docs/install/pi-management.md" - local lifecycle_contract="$root/docs/contracts/tht-pi.md" - [[ -f "$guide" ]] || { - echo "missing Pi management guide: docs/install/pi-management.md" >&2 - return 1 - } - [[ -f "$lifecycle_contract" ]] || { - echo "missing Pi lifecycle contract: docs/contracts/tht-pi.md" >&2 - return 1 - } - require_text "$lifecycle_contract" "Pi lifecycle contract" \ - "io.thothii.pi.version" - if grep -Fq 'org.opencontainers.image.version' "$lifecycle_contract"; then - echo "Pi lifecycle contract must use io.thothii.pi.version, not org.opencontainers.image.version" >&2 - return 1 - fi - require_headings "$guide" "Pi management guide" \ - "Choose application defaults" \ - "Edit the provider catalog and enabled-model policy" \ - "Store provider credentials" \ - "Reload changed configuration" \ - "Update the bundled Pi version" \ - "Recover a failed lifecycle operation" \ - "Direct support access" - require_text "$guide" "Pi management guide" \ - "pi status" \ - "pi doctor" \ - "pi test" \ - "pi check" \ - "pi configure" \ - "pi restart --yes --drain" \ - "restart only core" \ - "deploy/pi/models.json" \ - "deploy/pi/settings.json" \ - "policy only" \ - "backend installation settings" \ - "PI_AUTH_FILE" \ - "pi update" \ - "pi rollback --yes" \ - "pi maintenance status" \ - "pi maintenance recover --yes" \ - "pi logs" \ - "/run/secrets" \ - "Raw Compose access is unsupported" - if grep -Fq '~/.pi/agent/' "$guide"; then - echo "Pi management guide must not direct ThothII operators to native Pi paths" >&2 - return 1 - fi - if grep -Eqi 'browser shell|host-native Pi|host Pi|running container|live container' "$guide"; then - echo "Pi management guide must not include native-Pi, browser-shell, or live-container workflow" >&2 - return 1 - fi - node - "$guide" <<'NODE' -const fs = require("fs"); -const source = fs.readFileSync(process.argv[2], "utf8"); -if (/docker\s+compose(?:.|\n){0,160}\bexec\b(?:.|\n){0,80}\bcore\b(?:.|\n){0,80}\bpi\b/i.test(source)) { - throw new Error("raw non-installation-aware Compose Pi access is forbidden"); -} -const marker = "## Direct support access"; -const start = source.indexOf(marker); -const support = start < 0 ? "" : source.slice(start, source.indexOf("\n## ", start + marker.length) < 0 - ? source.length : source.indexOf("\n## ", start + marker.length)); -for (const token of ["unsupported", "tht", "pi status", "pi doctor", "pi test", "pi logs"]) { - if (!support.toLowerCase().includes(token.toLowerCase())) { - throw new Error(`direct support section lacks installation-aware diagnostic: ${token}`); - } -} -NODE - echo "Pi management guide contract passed" -} - -verify_server_guide() { - local guide="$root/docs/install/server.md" - [[ -f "$guide" ]] || { - echo "missing server installation guide sections: docs/install/server.md" >&2 - return 1 - } - require_headings "$guide" "server installation guide" \ - "Deployment contract" \ - "Service account and directories" \ - "Firewall and network boundaries" \ - "Address co-resident external services" \ - "Prepare operator files and secrets" \ - "Build locally or select pinned images" \ - "Install tht" \ - "Start and verify readiness" \ - "Configure TLS and upstream authentication" \ - "Operate Pi, drain, and roll back" \ - "Back up and restore" \ - "Diagnostics" \ - "Data-preserving uninstall" - require_text "$guide" "server installation guide" \ - "frontend" \ - "core" \ - "UID/GID 10001" \ - "does not require or permit creation" \ - "getent passwd 10001" \ - "getent group 10001" \ - "-m 0750 /srv/thothii/operator" \ - "chmod 0600 /srv/thothii/operator/server.env" \ - "THT_THT_OUTPUT_DIRECTORY=/srv/thothii/operator/build-output" \ - "/srv/thothii" \ - "example operator root" \ - "/run/secrets" \ - "Git-backed workspace registry is the source of truth" \ - "host.docker.internal" \ - "host-gateway" \ - "com.docker.network.bridge.name" \ - "DOCKER-USER" \ - "iptables -I INPUT" \ - "container 127.0.0.1" \ - "collection" \ - "embedding" \ - "bash scripts/build-local.sh" \ - "@sha256:" \ - "bash scripts/build-tht.sh" \ - "tht --installation" \ - "sessions migrate --yes" \ - '"pending":[]' \ - '"drifted":[]' \ - "remove --yes" \ - "THT_BACKUP_ROOT=/srv/thothii-backups" \ - "sha256sum --check SHA256SUMS" \ - "curl --fail http://127.0.0.1:8080/health" \ - "https://thoth.example.com" \ - "pi update" \ - "--drain" \ - "pi rollback --yes" \ - "pi maintenance recover --yes" \ - "docker compose down --volumes" \ - "reverse-proxy-nginx.md" \ - "reverse-proxy-caddy.md" - if ! grep -Fq 'sudo install -d -o "$operator_uid" -g 10001 -m 0750 /srv/thothii' "$guide"; then - echo "server installation guide does not set parent traversal boundary" >&2 - return 1 - fi - if grep -Eq '(^|[[:space:]])(sudo[[:space:]]+)?(useradd|groupadd|usermod)([[:space:]]|$)|sudo[[:space:]]+-u[[:space:]]+thothii|thothii-ops' "$guide"; then - echo "server installation guide creates or depends on a host identity" >&2 - return 1 - fi - node - "$guide" <<'NODE' -const fs = require("fs"); -const source = fs.readFileSync(process.argv[2], "utf8"); -if (/omics_portal|chirone|localllm_default|datamart-builder|compose\.production|compose\.psd-local/i.test(source)) { - throw new Error("server installation guide introduces forbidden application coupling"); -} -if (/\/var\/run\/docker\.sock|docker\.sock/i.test(source)) { - throw new Error("server installation guide introduces a Docker socket dependency"); -} -for (const line of source.split(/\n/)) { - const match = line.match(/^\s*([A-Z][A-Z0-9_]*(?:PASSWORD|TOKEN|API_KEY|SECRET)[A-Z0-9_]*)\s*=\s*(\S.*)$/); - if (!match) continue; - const [, name, rawValue] = match; - const value = rawValue.trim(); - if (!/(?:_FILE|_SOURCE)$/.test(name) && value && !/^\$\{?[A-Z_][A-Z0-9_]*\}?$/.test(value)) { - throw new Error("server installation guide embeds a secret value"); - } -} -let inCodeFence = false; -for (const line of source.split(/\n/)) { - if (line.trimStart().startsWith("```")) { - inCodeFence = !inCodeFence; - continue; - } - if (!line.includes("docker compose down --volumes")) continue; - const normalized = line.toLowerCase().replaceAll("*", ""); - if (inCodeFence || !/(do not|never)/.test(normalized) || /^\s*(docker|&?\s*docker)/.test(normalized)) { - throw new Error("server docker compose down --volumes must appear only in an explicit prose prohibition"); - } -} -if (/```(?:sh|bash)\n[\s\S]*?\bdocker\s+rm\b[\s\S]*?```/i.test(source)) { - throw new Error("server uninstall bypasses installation-aware removal"); -} -if (/host-gateway[^\n]{0,120}(?:listen|listening|bound)[^\n]{0,80}127\.0\.0\.1|(?:listen|listening|bound)[^\n]{0,80}127\.0\.0\.1[^\n]{0,120}host-gateway/i.test(source)) { - throw new Error("server host-gateway guidance assumes a host loopback listener"); -} -const pinnedStart = source.indexOf("## Build locally or select pinned images"); -const pinnedEnd = source.indexOf("\n## ", pinnedStart + 3); -const pinnedSection = source.slice(pinnedStart, pinnedEnd < 0 ? source.length : pinnedEnd); -const pinnedBlock = [...pinnedSection.matchAll(/```yaml\n([\s\S]*?)```/g)].map((match) => match[1]) - .find((block) => block.includes("session-migrate:")) || ""; -function pinnedService(name) { - const match = pinnedBlock.match(new RegExp(`^ ${name}:\\n((?: [^\\n]*\\n)+)`, "m")); - return match ? match[1] : ""; -} -const pinnedCore = pinnedService("core"); -const pinnedMigrator = pinnedService("session-migrate"); -const pinnedFrontend = pinnedService("frontend"); -const coreImage = pinnedCore.match(/image:\s*(\S+)/)?.[1]; -const migratorImage = pinnedMigrator.match(/image:\s*(\S+)/)?.[1]; -const frontendImage = pinnedFrontend.match(/image:\s*(\S+)/)?.[1]; -if (![pinnedCore, pinnedMigrator, pinnedFrontend].every((block) => block.includes("build: !reset null")) || - !coreImage || coreImage !== migratorImage || !/@sha256:<64-lowercase-hex-digits>$/.test(coreImage) || - !frontendImage || !/@sha256:<64-lowercase-hex-digits>$/.test(frontendImage)) { - throw new Error("server pinned image override must pin core, session-migrate, and frontend without builds"); -} -if (/session-migrate:[\s\S]{0,180}image:\s*thothii-core:local/.test(source) && - /core:[\s\S]{0,180}image:\s*registry\.[^\n]+@sha256:[a-f0-9]{64}/.test(source)) { - throw new Error("server pinned migration image must equal the pinned core image"); -} -if (/```(?:sh|bash)\n[\s\S]*?\bdocker compose\s+(?:up|stop|down|restart|pull|build)\b[\s\S]*?```/i.test(source)) { - throw new Error("server lifecycle must use tht, not raw Docker Compose"); -} -NODE - echo "server installation guide contract passed" -} - -verify_reverse_proxy_nginx_guide() { - local guide="$root/docs/install/reverse-proxy-nginx.md" - [[ -f "$guide" ]] || { - echo "missing Nginx reverse-proxy guide: docs/install/reverse-proxy-nginx.md" >&2 - return 1 - } - require_headings "$guide" "Nginx reverse-proxy guide" \ - "Trust boundary" \ - "Validate and reload" \ - "Test authentication and SSE" - require_text "$guide" "Nginx reverse-proxy guide" \ - "Forwarding identity headers alone does not authenticate a user" \ - "authentication gateway" \ - "2xx" \ - "TLS" \ - "frontend" - node - "$guide" <<'NODE' -const fs = require("fs"); -const source = fs.readFileSync(process.argv[2], "utf8"); -const block = [...source.matchAll(/```nginx\n([\s\S]*?)```/g)].map((match) => match[1]).join("\n"); -function stripNginxComments(text) { - let effective = ""; - let quote = null; - let escaped = false; - let comment = false; - for (const character of text) { - if (comment) { - if (character === "\n") { - effective += character; - comment = false; - } - continue; - } - if (escaped) { - effective += character; - escaped = false; - continue; - } - if (character === "\\") { - effective += character; - escaped = true; - continue; - } - if (quote !== null) { - effective += character; - if (character === quote) quote = null; - continue; - } - if (character === '"' || character === "'") { - effective += character; - quote = character; - continue; - } - if (character === "#") { - comment = true; - continue; - } - effective += character; - } - return effective; -} -const effectiveBlock = stripNginxComments(block); -const tokens = [ - "listen 443 ssl;", "ssl_certificate ", "ssl_certificate_key ", - "location = /_authenticate {", "internal;", "proxy_pass http://auth-gateway:4180/verify;", - "auth_request /_authenticate;", "proxy_pass http://127.0.0.1:8080;", - "proxy_http_version 1.1;", "proxy_buffering off;", "proxy_cache off;", - "proxy_read_timeout 3600s;", -]; -if (/127\.0\.0\.1:8787|\bcore:8787\b/.test(effectiveBlock) || !effectiveBlock.includes("http://127.0.0.1:8080")) { - throw new Error("Nginx proxy must forward only to frontend on 127.0.0.1:8080"); -} -for (const token of tokens) { - if (!effectiveBlock.includes(token)) throw new Error(`Nginx proxy lacks structural token: ${token}`); -} -if (/proxy_set_header\s+X-Thoth-Trusted-[^;]+\$http_/i.test(effectiveBlock)) { - throw new Error("Nginx proxy trusts a client-supplied identity header"); -} -const identities = [ - ["issuer", "Principal-Issuer", "thoth_principal_issuer", "x_thoth_principal_issuer"], - ["subject", "Principal-Subject", "thoth_principal_subject", "x_thoth_principal_subject"], - ["display", "Principal-Display-Name", "thoth_principal_display_name", "x_thoth_principal_display_name"], - ["admin", "Is-Admin", "thoth_is_admin", "x_thoth_is_admin"], -]; -function escaped(value) { return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); } -function nginxLocations(text) { - const locations = []; - const pattern = /\blocation\s+([^\n{]+)\{/g; - for (const match of text.matchAll(pattern)) { - const opening = match.index + match[0].lastIndexOf("{"); - let depth = 0; - let closing = -1; - for (let index = opening; index < text.length; index++) { - if (text[index] === "{") depth++; - if (text[index] === "}" && --depth === 0) { - closing = index; - break; - } - } - if (closing < 0) throw new Error(`Nginx proxy has unterminated location: ${match[1].trim()}`); - locations.push({selector: match[1].trim(), body: text.slice(opening + 1, closing)}); - } - return locations; -} -const locations = nginxLocations(effectiveBlock); -const frontendLocations = locations.filter((location) => - /proxy_pass\s+http:\/\/127\.0\.0\.1:8080\s*;/.test(location.body)); -if (frontendLocations.length === 0) { - throw new Error("Nginx proxy lacks a frontend upstream location"); -} -const authenticatedFrontendLocations = frontendLocations.filter((location) => - /auth_request\s+\/_authenticate\s*;/.test(location.body)); -const directFrontendLocations = frontendLocations.filter((location) => - !/auth_request\s+\/_authenticate\s*;/.test(location.body)); -if (directFrontendLocations.length > 1) { - throw new Error("Nginx direct OIDC mode contains an additional frontend bypass location"); -} -for (const frontendLocation of frontendLocations) { - for (const token of [ - "proxy_http_version 1.1;", "proxy_buffering off;", "proxy_cache off;", - "proxy_read_timeout 3600s;", - ]) { - if (!frontendLocation.body.includes(token)) { - throw new Error(`Nginx proxy lacks structural token: ${token}`); - } - } -} -if (authenticatedFrontendLocations.length > 0) { - const authLocations = locations.filter((location) => location.selector === "= /_authenticate"); - if (authLocations.length !== 1) { - throw new Error("Nginx proxy must define exactly one authentication location for upstream mode"); - } - const authLocation = authLocations[0].body; - for (const [label, publicName, variable, upstream] of identities) { - const trustedName = publicName === "Is-Admin" ? "Is-Admin" : publicName; - const publicClear = new RegExp(`proxy_set_header\\s+X-Thoth-${escaped(publicName)}\\s+"";`); - const trustedHeader = `X-Thoth-Trusted-${trustedName}`; - const trustedClear = new RegExp(`proxy_set_header\\s+${escaped(trustedHeader)}\\s+"";`); - const authPublicAt = authLocation.search(publicClear); - const authTrustedAt = authLocation.search(trustedClear); - if (authPublicAt < 0) { - throw new Error(`Nginx auth location does not clear inbound ${label} identity`); - } - if (authTrustedAt < 0) { - throw new Error(`Nginx auth location does not clear inbound trusted ${label} identity`); - } - for (const frontendLocation of authenticatedFrontendLocations) { - const frontendPublicAt = frontendLocation.body.search(publicClear); - if (frontendPublicAt < 0) { - throw new Error(`Nginx frontend location does not clear inbound ${label} identity`); - } - const normalizedFrontend = frontendLocation.body.replace(/\s+/g, " "); - const captureAt = normalizedFrontend.search(new RegExp(`auth_request_set\\s+\\$${variable}\\s+\\$upstream_http_${upstream};`)); - if (captureAt < 0) { - throw new Error(`Nginx frontend location does not capture authenticated ${label} identity`); - } - const mapAt = normalizedFrontend.search(new RegExp(`proxy_set_header\\s+${escaped(trustedHeader)}\\s+\\$${variable};`)); - if (mapAt < 0) { - throw new Error(`Nginx frontend location does not map authenticated ${label} identity`); - } - } - if (new RegExp(`auth_request_set\\s+\\$${variable}|proxy_set_header\\s+${escaped(trustedHeader)}\\s+\\$${variable};`).test(authLocation)) { - throw new Error(`Nginx auth location performs a forbidden ${label} capture or mapping`); - } - } -} else if (directFrontendLocations.length === 0) { - throw new Error("Nginx proxy lacks a direct or authenticated frontend path"); -} -for (const location of locations) { - const upstreams = [...location.body.matchAll(/proxy_pass\s+([^;]+);/g)].map((match) => match[1].trim()); - for (const upstream of upstreams) { - if (location.selector === "= /_authenticate" && upstream === "http://auth-gateway:4180/verify") continue; - if (upstream === "http://127.0.0.1:8080") continue; - throw new Error(`Nginx location proxies to an unreviewed upstream: ${upstream}`); - } -} -for (const frontendLocation of authenticatedFrontendLocations) { - if (!/auth_request\s+\/_authenticate\s*;/.test(frontendLocation.body)) { - throw new Error("Nginx authenticated frontend path bypasses complete authentication contract"); - } -} -NODE - echo "Nginx reverse-proxy guide contract passed" -} - -verify_reverse_proxy_caddy_guide() { - local guide="$root/docs/install/reverse-proxy-caddy.md" - [[ -f "$guide" ]] || { - echo "missing Caddy reverse-proxy guide: docs/install/reverse-proxy-caddy.md" >&2 - return 1 - } - require_headings "$guide" "Caddy reverse-proxy guide" \ - "Trust boundary" \ - "Validate and reload" \ - "Test authentication and SSE" - require_text "$guide" "Caddy reverse-proxy guide" \ - "Forwarding identity headers alone does not authenticate a user" \ - "authentication gateway" \ - "2xx" \ - "Caddy terminates TLS" \ - "frontend" - node - "$guide" <<'NODE' -const fs = require("fs"); -const source = fs.readFileSync(process.argv[2], "utf8"); -const block = [...source.matchAll(/```caddyfile\n([\s\S]*?)```/g)].map((match) => match[1]).join("\n"); -const tokens = [ - "thoth.example.invalid {", "route {", - "forward_auth auth-gateway:4180 {", "uri /verify", "copy_headers {", - "reverse_proxy 127.0.0.1:8080 {", "flush_interval -1", -]; -if (/127\.0\.0\.1:8787|\bcore:8787\b/.test(block) || !block.includes("127.0.0.1:8080")) { - throw new Error("Caddy proxy must forward only to frontend on 127.0.0.1:8080"); -} -for (const token of tokens) { - if (!block.includes(token)) throw new Error(`Caddy proxy lacks structural token: ${token}`); -} -function directiveBlock(text, marker) { - const start = text.indexOf(marker); - if (start < 0) throw new Error(`Caddy proxy lacks scoped block: ${marker}`); - const opening = text.indexOf("{", start); - let depth = 0; - for (let index = opening; index < text.length; index++) { - if (text[index] === "{") depth++; - if (text[index] === "}" && --depth === 0) return {start, end: index, body: text.slice(opening + 1, index)}; - } - throw new Error(`Caddy proxy has unterminated scoped block: ${marker}`); -} -const route = directiveBlock(block, "route {"); -const forward = directiveBlock(route.body, "forward_auth auth-gateway:4180 {"); -const forwardAt = route.body.indexOf("forward_auth auth-gateway:4180 {"); -for (const [label, publicName, trustedName] of [ - ["issuer", "X-Thoth-Principal-Issuer", "X-Thoth-Trusted-Principal-Issuer"], - ["subject", "X-Thoth-Principal-Subject", "X-Thoth-Trusted-Principal-Subject"], - ["display", "X-Thoth-Principal-Display-Name", "X-Thoth-Trusted-Principal-Display-Name"], - ["admin", "X-Thoth-Is-Admin", "X-Thoth-Trusted-Is-Admin"], -]) { - const publicClearAt = route.body.indexOf(`request_header -${publicName}`); - if (publicClearAt < 0) { - throw new Error(`Caddy proxy does not clear inbound ${label} identity`); - } - const trustedClearAt = route.body.indexOf(`request_header -${trustedName}`); - if (trustedClearAt < 0) { - throw new Error(`Caddy proxy does not clear inbound trusted ${label} identity`); - } - if (publicClearAt > forwardAt || trustedClearAt > forwardAt) { - throw new Error("Caddy identity clears must precede forward_auth"); - } - if (!forward.body.includes(`${publicName}>${trustedName}`)) { - throw new Error(`Caddy proxy does not map authenticated ${label} identity`); - } -} -const outsideForward = route.body.slice(0, forward.start) + route.body.slice(forward.end + 1); -if (/X-Thoth-(?:Principal-[^\s>]+|Is-Admin)>X-Thoth-Trusted-/.test(outsideForward)) { - throw new Error("Caddy maps identity outside the authenticated response stage"); -} -NODE - echo "Caddy reverse-proxy guide contract passed" -} - -verify_caddy_adapted_identity_order() { - local adapted="$1" - node - "$adapted" <<'NODE' -const fs = require("fs"); -const document = JSON.parse(fs.readFileSync(process.argv[2], "utf8")); -const publicHeaders = [ - "X-Thoth-Principal-Issuer", "X-Thoth-Principal-Subject", - "X-Thoth-Principal-Display-Name", "X-Thoth-Is-Admin", -]; -const trustedHeaders = [ - "X-Thoth-Trusted-Principal-Issuer", "X-Thoth-Trusted-Principal-Subject", - "X-Thoth-Trusted-Principal-Display-Name", "X-Thoth-Trusted-Is-Admin", -]; -function authUpstream(handler) { - return handler?.handler === "reverse_proxy" && - (handler.upstreams || []).some((upstream) => upstream.dial === "auth-gateway:4180"); -} -function frontendUpstream(handler) { - return handler?.handler === "reverse_proxy" && - (handler.upstreams || []).some((upstream) => upstream.dial === "127.0.0.1:8080"); -} -function collectTrustedSets(value, collected = new Map()) { - if (!value || typeof value !== "object") return collected; - if (value.handler === "headers") { - for (const [name, replacement] of Object.entries(value.request?.set || {})) { - if (trustedHeaders.includes(name)) collected.set(name, replacement); - } - } - for (const child of Object.values(value)) collectTrustedSets(child, collected); - return collected; -} -const expectedClears = [...publicHeaders, ...trustedHeaders]; -function validateAuthenticatedMappings(auth) { - const successResponse = (auth.handle_response || []).find((response) => - (response.match?.status_code || []).map(Number).includes(2)); - if (!successResponse) throw new Error("Caddy adapted identity mapping is not restricted to auth 2xx"); - const mappings = collectTrustedSets(successResponse); - for (let index = 0; index < trustedHeaders.length; index++) { - const replacement = mappings.get(trustedHeaders[index]); - const expected = `{http.reverse_proxy.header.${publicHeaders[index]}}`; - if (!Array.isArray(replacement) || replacement.length !== 1 || replacement[0] !== expected) { - throw new Error(`Caddy adapted authenticated mapping is invalid for ${trustedHeaders[index]}`); - } - } -} -function validateFrontendPath(handlers) { - let authAt = -1; - for (let index = handlers.length - 1; index >= 0; index--) { - if (authUpstream(handlers[index])) { - authAt = index; - break; - } - } - if (authAt < 0) { - throw new Error("Caddy adapted frontend path bypasses complete authentication contract"); - } - for (const header of expectedClears) { - const clearAt = handlers.findIndex((handler) => - handler?.handler === "headers" && (handler.request?.delete || []).includes(header)); - if (clearAt < 0 || clearAt >= authAt) { - throw new Error("Caddy adapted identity clears must execute before authentication"); - } - } - validateAuthenticatedMappings(handlers[authAt]); - for (let index = 0; index < handlers.length; index++) { - if (index !== authAt && collectTrustedSets(handlers[index]).size !== 0) { - throw new Error("Caddy adapted config maps trusted identity outside auth success"); - } - } -} -let frontendPaths = 0; -function walk(value, inherited = []) { - if (!value || typeof value !== "object") return; - if (Array.isArray(value)) { - for (const child of value) walk(child, inherited); - return; - } - if (frontendUpstream(value)) { - frontendPaths++; - validateFrontendPath(inherited); - } - if (Array.isArray(value.handle)) { - const previous = []; - for (const handler of value.handle) { - walk(handler, [...inherited, ...previous]); - previous.push(handler); - } - for (const [key, child] of Object.entries(value)) { - if (key !== "handle") walk(child, inherited); - } - return; - } - const childContext = authUpstream(value) ? [...inherited, value] : inherited; - for (const child of Object.values(value)) walk(child, childContext); -} -walk(document); -if (frontendPaths === 0) { - throw new Error("Caddy adapted config lacks a frontend handler path"); -} -NODE -} - -verify_caddy_effective_proxy_guide() { - local adapted - adapted="$(mktemp "${tmp_prefix}thoth-caddy-adapted.XXXXXX")" - if ! awk ' - /^```caddyfile$/ { code=1; next } - code && /^```$/ { exit } - code { print } - ' "$root/docs/install/reverse-proxy-caddy.md" \ - | docker run --rm -i caddy:2.10.2-alpine caddy adapt --config - --adapter caddyfile >"$adapted"; then - rm -f "$adapted" - echo "Caddy documented configuration could not be adapted" >&2 - return 1 - fi - verify_caddy_adapted_identity_order "$adapted" - rm -f "$adapted" - echo "Caddy adapted trust-stage contract passed" -} - -verify_manual() { - local profile="$1" manual - manual="$root/docs/install/$profile-workspace-registry.md" - local -a headings - if [[ "$profile" == local ]]; then - headings=( - "Prerequisites" - "Prepare and publish a workspace source" - "Configure the remote Git repository" - "Start and update the installation" - "Complete runtime secrets in Workspace management" - "Validation and activation behavior" - "Backup, rotation, and recovery" - "Troubleshooting" - ) - else - headings=( - "Service account, storage, and firewall" - "Prepare and publish a workspace source" - "Configure the remote Git repository" - "Start and update the installation" - "Complete runtime secrets in Workspace management" - "Validation and activation behavior" - "Backup, rotation, and recovery" - "Troubleshooting" - ) - fi - for heading in "${headings[@]}"; do - grep -Fqx "## $heading" "$manual" || { - echo "missing required heading in $profile manual: $heading" >&2 - return 1 - } - done - local -a expected_steps - if [[ "$profile" == local ]]; then - expected_steps=( - 'export THT_SOURCE_ROOT=/absolute/path/to/ThothII' - 'thothii-installation.yaml' - 'workspaceRepository' - '"$THT_BIN" --installation "$INSTALLATION" start' - '"$THT_BIN" --installation "$INSTALLATION" doctor' - ) - else - expected_steps=( - 'THT_BIN=/srv/thothii/operator/tht' - 'INSTALLATION=/srv/thothii/operator/thothii-installation.yaml' - '"$THT_BIN" --installation "$INSTALLATION" start' - '"$THT_BIN" --installation "$INSTALLATION" doctor' - 'docs/install/examples/thothii-installation.server.yaml' - 'compose.yaml' - 'deploy/compose.server.yaml' - 'server.md' - ) - fi - for expected in "${expected_steps[@]}"; do - grep -Fq -- "$expected" "$manual" || { - echo "$profile manual lacks canonical operator step: $expected" >&2 - return 1 - } - done - if rg -n 'local-compose\.workspace-registry|server-compose\.workspace-registry|connector-secrets\.workspace-registry|docker compose' "$manual"; then - echo "$profile manual documents a superseded or bypassed Compose path" >&2 - return 1 - fi - verify_path_variable_values "$manual" - echo "$profile manual canonical base+override references passed" -} - -verify_read_only_workspace_runtime_contract() { - python3 - "$root" <<'PY' -import pathlib, sys, yaml - -root = pathlib.Path(sys.argv[1]) -compose = yaml.safe_load((root / "compose.yaml").read_text()) -services = compose["services"] -core = services["core"] -maintenance = services["workspace-maintenance"] -environment = core["environment"] - -for forbidden in ("THT_WORKSPACE_GIT_AUTHOR_NAME", "THT_WORKSPACE_GIT_AUTHOR_EMAIL"): - if forbidden in environment: - raise SystemExit(f"compose retains Git write identity: {forbidden}") -for key, value in { - "THT_WORKSPACE_SECRET_STORE_ROOT": "/data/workspace-secrets", - "THT_WORKSPACE_SECRET_RUNTIME_ROOT": "/tmp/thothii-workspace-secrets", -}.items(): - if environment.get(key) != value or maintenance["environment"].get(key) != value: - raise SystemExit(f"workspace secret setting missing from core/maintenance: {key}") -if "workspace-secrets" not in compose["volumes"]: - raise SystemExit("workspace-secrets persistent volume is missing") -if not any("workspace-secrets:/data/workspace-secrets" in str(value) for value in core["volumes"]): - raise SystemExit("core does not persist the workspace secret vault") -if not any(mount.get("source") == "workspace-secrets" and mount.get("target") == "/data/workspace-secrets" - for mount in maintenance["volumes"] if isinstance(mount, dict)): - raise SystemExit("workspace-maintenance cannot use the encrypted workspace vault") -dockerfile = (root / "docker/core.Dockerfile").read_text() -if "/data/workspace-secrets" not in dockerfile: - raise SystemExit("core image does not pre-create the workspace secret volume target") - -checked = [ - root / "docs/install/local-workspace-registry.md", - root / "docs/install/server-workspace-registry.md", - root / "docs/install/local.md", - root / "docs/install/server.md", - root / "docs/install/psd-workspace-setup.md", - root / "docs/guida-utente.md", - root / "deploy/workspace-registry.env.example", - root / "deploy/env/local.env.example", - root / "deploy/env/server.env.example", - root / "deploy/psd/operator.env.example", - root / "docs/install/examples/thothii-installation.local.yaml", - root / "docs/install/examples/thothii-installation.server.yaml", -] -joined = "\n".join(path.read_text() for path in checked) -for forbidden in ( - "THT_WORKSPACE_GIT_AUTHOR_NAME", - "THT_WORKSPACE_GIT_AUTHOR_EMAIL", - "connector-secrets.local.yaml", - "connector-secrets.server.yaml", - "THT_WORKSPACE_BINDINGS_ENV_FILE", - "POST /workspaces/publish", - "POST /workspaces/import", - "Import workspace bundle", -): - if forbidden in joined: - raise SystemExit(f"active workspace documentation retains obsolete contract: {forbidden}") -for required in ( - "GitHub, GitLab, or Gitea", - "read-only consumer", - "workspace-secrets", - "write-only", - "previous active revision", -): - if required.lower() not in joined.lower(): - raise SystemExit(f"active workspace documentation lacks required concept: {required}") - -ui = (root / "frontend/src/shell/WorkspaceManager.tsx").read_text() -for required in ( - "Update workspace repository", - "No workspace selection is required", - "deletes temporary files when the check finishes", -): - if required not in ui: - raise SystemExit(f"Workspace management lacks required explanation: {required}") -for forbidden in ("Import bundle", "Export bundle", "localStorage"): - if forbidden in ui: - raise SystemExit(f"Workspace management retains obsolete behavior: {forbidden}") -PY - echo "read-only workspace repository and encrypted runtime-secret contract passed" -} - -verify_local_installation_example() { - local example="$root/docs/install/examples/thothii-installation.local.yaml" - [[ -f "$example" ]] || { - echo "missing local installation example: docs/install/examples/thothii-installation.local.yaml" >&2 - return 1 - } - - local fixture source_copy operator_dir copied_example env_file auth_config_root - fixture="$(mktemp -d "${tmp_prefix}thoth local install.XXXXXX")" - trap 'rm -rf "$fixture"' RETURN - [[ "$fixture" == *" "* ]] || { - echo "local installation fixture path does not contain spaces" >&2 - return 1 - } - source_copy="$fixture/ThothII source" - operator_dir="$fixture/operator files" - auth_config_root="$operator_dir/auth config" - mkdir -p "$source_copy/deploy/pi" "$operator_dir" "$auth_config_root" - cp "$root/compose.yaml" "$source_copy/compose.yaml" - cp "$root/deploy/compose.local.yaml" "$source_copy/deploy/compose.local.yaml" - cp "$root/deploy/compose.git-ssh.yaml" "$source_copy/deploy/compose.git-ssh.yaml" - cp "$root/deploy/pi/models.json" "$source_copy/deploy/pi/models.json" - cp "$root/deploy/pi/settings.json" "$source_copy/deploy/pi/settings.json" - - write_private "$operator_dir/pi-auth.json" '{"zai":{"type":"api_key","key":"fixture-local-pi-key"}}' - write_private "$operator_dir/thothii.secrets" 'THT_MODEL_API_KEY=fixture-local-model-key' - write_private "$operator_dir/git-ssh-key" 'fixture-local-ssh-key' - write_private "$operator_dir/git-known-hosts" 'fixture-local-known-hosts' - env_file="$source_copy/deploy/env/local.env" - mkdir -p "$source_copy/deploy/env" - printf '%s\n' \ - 'THT_WORKSPACE_GIT_REMOTE=ssh://git@git.example.invalid/platform/thoth-workspaces.git' \ - "PI_AUTH_FILE=$operator_dir/pi-auth.json" \ - "THT_SECRETS_FILE=$operator_dir/thothii.secrets" \ - "THT_WORKSPACE_GIT_SSH_KEY_FILE=$operator_dir/git-ssh-key" \ - "THT_WORKSPACE_GIT_KNOWN_HOSTS_FILE=$operator_dir/git-known-hosts" \ - "THT_AUTH_CONFIG_ROOT=$auth_config_root" \ - >"$env_file" - - copied_example="$fixture/thothii-installation.yaml" - local contents - contents="$(<"$example")" - contents="${contents//\/absolute\/path\/to\/ThothII/$source_copy}" - contents="${contents//\/absolute\/path\/to\/thothii-operator/$operator_dir}" - printf '%s\n' "$contents" >"$copied_example" - - local profile project_directory descriptor_env value - local -a overrides files - profile="$(sed -n 's/^profile: \([^[:space:]]*\)$/\1/p' "$copied_example")" - project_directory="$(sed -n 's/^projectDirectory: "\(.*\)"$/\1/p' "$copied_example")" - descriptor_env="$(sed -n 's/^envFile: "\(.*\)"$/\1/p' "$copied_example")" - while IFS= read -r value; do overrides+=("$value"); done < <(sed -n 's/^ - "\(.*\)"$/\1/p' "$copied_example") - [[ "$profile" == local && "$project_directory" == "$source_copy" && "$descriptor_env" == "$env_file" ]] || { - echo "local installation example does not resolve its required fields" >&2 - return 1 - } - [[ "${#overrides[@]}" -eq 1 && "${overrides[0]}" == "$source_copy/deploy/compose.git-ssh.yaml" ]] || { - echo "local installation example does not select the expected optional overrides" >&2 - return 1 - } - files=(-f "$project_directory/compose.yaml" -f "$project_directory/deploy/compose.$profile.yaml") - for value in "${overrides[@]}"; do files+=(-f "$value"); done - local rendered="$fixture/local-installation.json" - "$root/scripts/compose-with-preflight.sh" --env-file "$descriptor_env" \ - "${files[@]}" config --format json >"$rendered" - node - "$rendered" "$auth_config_root" <<'NODE' -const fs = require("fs"); -const [path, authConfigRoot] = process.argv.slice(2); -const config = JSON.parse(fs.readFileSync(path, "utf8")); -if (Object.keys(config.services).sort().join(",") !== "catalog-db,core,embedding,embedding-model-init,frontend,qdrant") { - throw new Error("local installation example must render the internal semantic stack"); -} -const authMount = (config.services.core.volumes || []).find( - (mount) => mount.target === "/run/thothii-auth", -); -if (!authMount || authMount.source !== authConfigRoot || authMount.read_only !== true) { - throw new Error("local installation example must mount its fixture auth root read-only"); -} -const output = JSON.stringify(config); -for (const secret of [ - "fixture-local-pi-key", - "fixture-local-model-key", - "fixture-local-ssh-key", - "fixture-local-known-hosts", - "fixture-local-dwh-password", -]) { - if (output.includes(secret)) throw new Error("local installation rendering exposed a fixture secret"); -} -NODE - echo "local installation example rendered from path with spaces passed" -} - -verify_server_installation_example() { - local example="$root/docs/install/examples/thothii-installation.server.yaml" - [[ -f "$example" ]] || { - echo "missing server installation example: docs/install/examples/thothii-installation.server.yaml" >&2 - return 1 - } - - local fixture source_copy operator_dir copied_example env_file backup_root auth_config_root - fixture="$(mktemp -d "${tmp_prefix}thoth server install.XXXXXX")" - trap 'rm -rf "$fixture"' RETURN - [[ "$fixture" == *" "* ]] || { - echo "server installation fixture path does not contain spaces" >&2 - return 1 - } - source_copy="$fixture/ThothII server source" - operator_dir="$fixture/server operator files" - backup_root="$fixture/server backups" - auth_config_root="$operator_dir/auth config" - mkdir -p "$source_copy/deploy/pi" "$source_copy/deploy/workspaces" \ - "$operator_dir/data/workspace-secrets" "$operator_dir/pi-state" "$operator_dir/workspace-registry" \ - "$auth_config_root" "$backup_root" - "$root/scripts/prepare-server-pi-state.sh" \ - "$operator_dir/pi-state" "$(id -u)" "$(id -g)" >/dev/null - cp "$root/compose.yaml" "$source_copy/compose.yaml" - cp "$root/deploy/compose.server.yaml" "$source_copy/deploy/compose.server.yaml" - cp "$root/deploy/compose.session-server.yaml.example" \ - "$source_copy/deploy/compose.session-server.yaml.example" - cp "$root/deploy/compose.git-ssh.yaml" "$source_copy/deploy/compose.git-ssh.yaml" - cp "$root/deploy/pi/models.json" "$source_copy/deploy/pi/models.json" - cp "$root/deploy/pi/settings.json" "$source_copy/deploy/pi/settings.json" - cp "$root/deploy/workspaces/server-sessions.yaml.example" \ - "$source_copy/deploy/workspaces/server-sessions.yaml.example" - - write_private "$operator_dir/pi-auth.json" '{"zai":{"type":"api_key","key":"fixture-server-pi-key"}}' - write_private "$operator_dir/thothii.secrets" 'THT_MODEL_API_KEY=fixture-server-model-key' - write_private "$operator_dir/git-ssh-key" 'fixture-server-ssh-key' - write_private "$operator_dir/git-known-hosts" 'fixture-server-known-hosts' - write_private "$operator_dir/session-runtime-password" 'fixture-server-session-runtime-password' - write_private "$operator_dir/session-migrator-password" 'fixture-server-session-migrator-password' - write_private "$operator_dir/session-ca.pem" 'fixture-server-session-ca' - env_file="$operator_dir/server.env" - printf '%s\n' \ - 'THOTH_SERVER_BIND=127.0.0.1' \ - 'THOTH_HTTP_PORT=8080' \ - 'THT_WORKSPACE_GIT_REMOTE=ssh://git@git.example.invalid/platform/thoth-workspaces.git' \ - 'THT_WORKSPACE_GIT_BRANCH=main' \ - "PI_AUTH_FILE=$operator_dir/pi-auth.json" \ - "THT_SECRETS_FILE=$operator_dir/thothii.secrets" \ - "THT_WORKSPACE_GIT_SSH_KEY_FILE=$operator_dir/git-ssh-key" \ - "THT_WORKSPACE_GIT_KNOWN_HOSTS_FILE=$operator_dir/git-known-hosts" \ - "THT_DATA_ROOT=$operator_dir/data" \ - "THT_PI_STATE_ROOT=$operator_dir/pi-state" \ - "THT_WORKSPACE_REGISTRY_ROOT=$operator_dir/workspace-registry" \ - "THT_BACKUP_ROOT=$backup_root" \ - "THT_AUTH_CONFIG_ROOT=$auth_config_root" \ - "THT_SERVER_WORKSPACE_CONFIG=$source_copy/deploy/workspaces/server-sessions.yaml.example" \ - 'THT_LLM_URL=https://llm.example.invalid' \ - 'THT_SESSION_DB_HOST=sessions.example.invalid' \ - 'THT_SESSION_DB_NAME=thoth_sessions' \ - 'THT_SESSION_RUNTIME_USER=thoth_sessions_app' \ - 'THT_SESSION_MIGRATOR_USER=thoth_sessions_migrate' \ - "THT_SESSION_RUNTIME_PASSWORD_SOURCE=$operator_dir/session-runtime-password" \ - "THT_SESSION_MIGRATOR_PASSWORD_SOURCE=$operator_dir/session-migrator-password" \ - "THT_SESSION_CA_SOURCE=$operator_dir/session-ca.pem" \ - >"$env_file" - - copied_example="$fixture/thothii-installation.yaml" - local contents - contents="$(<"$example")" - contents="${contents//\/absolute\/path\/to\/ThothII/$source_copy}" - contents="${contents//\/absolute\/path\/to\/thothii-server-operator/$operator_dir}" - printf '%s\n' "$contents" >"$copied_example" - - local profile project_directory descriptor_env value - local -a overrides files - profile="$(sed -n 's/^profile: \([^[:space:]]*\)$/\1/p' "$copied_example")" - project_directory="$(sed -n 's/^projectDirectory: "\(.*\)"$/\1/p' "$copied_example")" - descriptor_env="$(sed -n 's/^envFile: "\(.*\)"$/\1/p' "$copied_example")" - while IFS= read -r value; do overrides+=("$value"); done < <(sed -n 's/^ - "\(.*\)"$/\1/p' "$copied_example") - [[ "$profile" == server && "$project_directory" == "$source_copy" && "$descriptor_env" == "$env_file" ]] || { - echo "server installation example does not resolve its required fields" >&2 - return 1 - } - [[ "${#overrides[@]}" -eq 2 && "${overrides[0]}" == "$source_copy/deploy/compose.session-server.yaml.example" \ - && "${overrides[1]}" == "$source_copy/deploy/compose.git-ssh.yaml" ]] || { - echo "server installation example does not select the expected optional overrides" >&2 - return 1 - } - files=(-f "$project_directory/compose.yaml" -f "$project_directory/deploy/compose.$profile.yaml") - for value in "${overrides[@]}"; do files+=(-f "$value"); done - local rendered="$fixture/server-installation.json" - "$root/scripts/compose-with-preflight.sh" --env-file "$descriptor_env" \ - "${files[@]}" config --format json >"$rendered" - node - "$rendered" "$auth_config_root" <<'NODE' -const fs = require("fs"); -const [path, authConfigRoot] = process.argv.slice(2); -const config = JSON.parse(fs.readFileSync(path, "utf8")); -if (Object.keys(config.services).sort().join(",") !== "catalog-db,core,embedding,embedding-model-init,frontend,qdrant") { - throw new Error("server installation example must render the internal semantic stack"); -} -const core = config.services.core; -const frontend = config.services.frontend; -const authMount = (core.volumes || []).find((mount) => mount.target === "/run/thothii-auth"); -if (!authMount || authMount.source !== authConfigRoot || authMount.read_only !== true) { - throw new Error("server installation example must mount its fixture auth root read-only"); -} -if (core.environment?.THOTH_PUBLIC_EXPOSURE !== "true" || - core.environment?.THT_AUTH_CONFIG_FILE !== "/run/thothii-auth/auth.yaml") { - throw new Error("server installation example must expose only the authenticated frontend"); -} -if ((core.ports || []).length !== 0) throw new Error("server installation example published core"); -const ports = frontend.ports || []; -if (ports.length !== 1 || ports[0].host_ip !== "127.0.0.1" || Number(ports[0].target) !== 8080) { - throw new Error("server installation example must publish only loopback frontend"); -} -const rendered = JSON.stringify(config); -if (/omics_portal|chirone|localllm_default|datamart-builder/i.test(rendered)) { - throw new Error("server installation example contains application coupling"); -} -for (const secret of [ - "fixture-server-pi-key", "fixture-server-model-key", "fixture-server-ssh-key", - "fixture-server-known-hosts", "fixture-server-dwh-password", - "fixture-server-session-runtime-password", "fixture-server-session-migrator-password", - "fixture-server-session-ca", -]) { - if (rendered.includes(secret)) throw new Error("server installation rendering exposed a fixture secret"); -} -NODE - echo "server installation example rendered from path with spaces passed" - - local migration_rendered="$fixture/server-migration.json" - "$root/scripts/compose-with-preflight.sh" --env-file "$descriptor_env" \ - "${files[@]}" --profile session-migrate config --format json >"$migration_rendered" - node - "$migration_rendered" <<'NODE' -const fs = require("fs"); -const config = JSON.parse(fs.readFileSync(process.argv[2], "utf8")); -const services = config.services || {}; -if (!services.core || !services["session-migrate"]) throw new Error("server migration profile is missing core or session-migrate"); -if (services.core.image !== services["session-migrate"].image) throw new Error("source migration image differs from core"); -if (services["session-migrate"].build) throw new Error("source migration service unexpectedly declares a build"); -NODE - - local pinned_template="$fixture/pinned-template.yaml" pinned_override="$operator_dir/pinned-images.yaml" - awk ' - /^## Build locally or select pinned images$/ { section=1; next } - section && /^```yaml$/ { code=1; next } - code && /^```$/ { exit } - code { print } - ' "$root/docs/install/server.md" >"$pinned_template" - sed \ - -e "s#registry.example.com/thothii/core@sha256:<64-lowercase-hex-digits>#registry.example.com/thothii/core@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa#g" \ - -e "s#registry.example.com/thothii/frontend@sha256:<64-lowercase-hex-digits>#registry.example.com/thothii/frontend@sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb#g" \ - "$pinned_template" >"$pinned_override" - chmod 0600 "$pinned_override" - local pinned_rendered="$fixture/server-pinned-migration.json" - "$root/scripts/compose-with-preflight.sh" --env-file "$descriptor_env" \ - "${files[@]}" -f "$pinned_override" --profile session-migrate config --format json >"$pinned_rendered" - node - "$pinned_rendered" <<'NODE' -const fs = require("fs"); -const config = JSON.parse(fs.readFileSync(process.argv[2], "utf8")); -const core = config.services?.core; -const frontend = config.services?.frontend; -const migrator = config.services?.["session-migrate"]; -if (!core || !frontend || !migrator) throw new Error("pinned migration profile lacks core, frontend, or session-migrate"); -if (core.image !== migrator.image || !/@sha256:[a-f0-9]{64}$/.test(core.image)) { - throw new Error("pinned migration image does not equal the exact core digest"); -} -if (!/@sha256:[a-f0-9]{64}$/.test(frontend.image)) throw new Error("frontend is not pinned by exact digest"); -for (const [name, service] of Object.entries({core, frontend, migrator})) { - if (service.build) throw new Error(name + " retained a local build in pinned mode"); - if (/:local$/.test(service.image || "")) throw new Error(name + " retained a local image in pinned mode"); -} -NODE - echo "server pinned migration image fixture passed" - - local checksum_root="$fixture/root-only-checksum" - mkdir -m 0700 "$checksum_root" - printf 'fixture backup bytes\n' >"$checksum_root/runtime-data.tgz" - /bin/sh -ceu 'cd "$1"; sha256sum runtime-data.tgz > SHA256SUMS; sha256sum --check SHA256SUMS' sh "$checksum_root" >/dev/null - printf 'corruption\n' >>"$checksum_root/runtime-data.tgz" - if (cd "$checksum_root" && sha256sum --check SHA256SUMS) >/dev/null 2>&1; then - echo "corrupted server backup checksum fixture was accepted" >&2 - return 1 - fi - echo "server backup checksum root-only fixture passed" -} - -write_private() { - local path="$1" value="$2" - printf '%s\n' "$value" >"$path" - chmod 0600 "$path" -} - -verify_compose_fixtures() { - local fixture profile rendered auth_config_root - fixture="$(mktemp -d "${tmp_prefix}thoth-install-fixtures.XXXXXX")" - trap 'rm -rf "$fixture"' RETURN - auth_config_root="$fixture/auth-config" - mkdir -p "$fixture/data/workspace-secrets" "$fixture/pi-state" \ - "$fixture/workspace-registry" "$auth_config_root" - - write_private "$fixture/pi-auth.json" '{"zai":{"type":"api_key","key":"fixture-native-auth-key"}}' - write_private "$fixture/thothii.secrets" 'THT_MODEL_API_KEY=fixture-model-api-key' - write_private "$fixture/git-ssh-key" 'fixture-git-ssh-key' - write_private "$fixture/git-known-hosts" 'fixture-git-known-hosts' - write_private "$fixture/session-runtime-password" 'fixture-session-runtime-password' - write_private "$fixture/session-migrator-password" 'fixture-session-migrator-password' - write_private "$fixture/session-ca.pem" 'fixture-session-ca' - cp "$root/deploy/workspaces/server-sessions.yaml.example" "$fixture/server-sessions.yaml" - printf '%s\n' \ - 'THT_WORKSPACE_GIT_REMOTE=ssh://git@git.example.invalid/platform/thoth-workspaces.git' \ - "PI_AUTH_FILE=$fixture/pi-auth.json" \ - "THT_SECRETS_FILE=$fixture/thothii.secrets" \ - "THT_WORKSPACE_GIT_SSH_KEY_FILE=$fixture/git-ssh-key" \ - "THT_WORKSPACE_GIT_KNOWN_HOSTS_FILE=$fixture/git-known-hosts" \ - "THT_DATA_ROOT=$fixture/data" \ - "THT_PI_STATE_ROOT=$fixture/pi-state" \ - "THT_WORKSPACE_REGISTRY_ROOT=$fixture/workspace-registry" \ - "THT_AUTH_CONFIG_ROOT=$auth_config_root" \ - "THT_SERVER_WORKSPACE_CONFIG=$fixture/server-sessions.yaml" \ - 'THT_SESSION_DB_HOST=sessions.example.invalid' \ - 'THT_SESSION_DB_NAME=thoth_sessions' \ - 'THT_SESSION_RUNTIME_USER=thoth_sessions_app' \ - 'THT_SESSION_MIGRATOR_USER=thoth_sessions_migrate' \ - "THT_SESSION_RUNTIME_PASSWORD_SOURCE=$fixture/session-runtime-password" \ - "THT_SESSION_MIGRATOR_PASSWORD_SOURCE=$fixture/session-migrator-password" \ - "THT_SESSION_CA_SOURCE=$fixture/session-ca.pem" \ - >"$fixture/operator.env" - - for profile in local server; do - rendered="$fixture/$profile.json" - files=( - -f "$root/compose.yaml" - -f "$root/deploy/compose.$profile.yaml" - ) - if [[ "$profile" == server ]]; then - files+=(-f "$root/deploy/compose.session-server.yaml.example") - fi - files+=( - -f "$root/deploy/compose.git-ssh.yaml" - ) - "$root/scripts/compose-with-preflight.sh" --env-file "$fixture/operator.env" \ - "${files[@]}" config --format json >"$rendered" - - node - "$rendered" "$profile" "$auth_config_root" <<'NODE' -const fs = require("fs"); -const [path, profile, authConfigRoot] = process.argv.slice(2); -const config = JSON.parse(fs.readFileSync(path, "utf8")); -if (Object.keys(config.services).sort().join(",") !== "catalog-db,core,embedding,embedding-model-init,frontend,qdrant") { - throw new Error(profile + ": mandatory stack must include the internal semantic services"); -} -const core = config.services.core; -const authMount = (core.volumes || []).find((mount) => mount.target === "/run/thothii-auth"); -if (!authMount || authMount.source !== authConfigRoot || authMount.read_only !== true) { - throw new Error(profile + ": core must mount its fixture auth root read-only"); -} -for (const target of [ - "/home/thoth/.pi/agent/auth.json", - "/home/thoth/.pi/agent/models.json", - "/home/thoth/.pi/agent/settings.json", -]) { - if (!(core.volumes || []).some((mount) => mount.target === target && mount.read_only)) { - throw new Error(profile + ": missing read-only Pi mount " + target); - } -} -const secretTargets = new Set((core.secrets || []).map((secret) => secret.target)); -for (const target of [ - "thothii.secrets", -]) { - if (!secretTargets.has(target)) throw new Error(profile + ": missing secret target " + target); -} -if (profile === "server") { - for (const target of ["session_runtime_password", "session_ca.pem"]) { - if (!secretTargets.has(target)) throw new Error("server: missing session secret " + target); - } -} -if ((config.services.frontend.secrets || []).length !== 0) { - throw new Error(profile + ": frontend received a runtime secret"); -} -const rendered = JSON.stringify(config); -for (const value of [ - "fixture-native-auth-key", "fixture-model-api-key", "fixture-git-ssh-key", - "fixture-git-known-hosts", - "fixture-session-runtime-password", "fixture-session-migrator-password", "fixture-session-ca", -]) { - if (rendered.includes(value)) throw new Error(profile + ": rendered Compose leaked " + value); -} -NODE - echo "canonical $profile base+override fixture passed" + for text in \ + 'workspace preprocess dwh' \ + 'workspace preprocess evidence' \ + 'workspace preprocess run' \ + 'workspace schema accept' \ + 'write-only runtime secrets' \ + 'Runtime sessions support direct PostgreSQL and REST bindings.'; do + require_text "$workspace" "$text" done - printf 'THT_WS_EXAMPLE_DWH_PASSWORD_SOURCE=relative/secret\n' >"$fixture/unsafe.env" - if verify_path_variable_values "$fixture/unsafe.env" >/dev/null 2>&1; then - echo "relative secret-source fixture was accepted" >&2 - return 1 - fi - echo "relative secret-source fixture rejected passed" + for obsolete in \ + 'docs/install/local-workspace-registry.md' \ + 'docs/install/server-workspace-registry.md' \ + 'docs/install/local.md' \ + 'docs/install/server.md'; do + forbid_text README.md "$obsolete" + forbid_text "$install" "$obsolete" + forbid_text "$workspace" "$obsolete" + done + echo "Local installation and workspace operations contract passed" } -case "$mode" in +verify_examples() { + require_file docs/install/examples/thothii-installation.local.yaml + require_file docs/install/examples/thothii-installation.server.yaml + require_file docs/install/examples/workspace-bindings.env.example + python3 - "$root/docs/install/examples/thothii-installation.local.yaml" \ + "$root/docs/install/examples/thothii-installation.server.yaml" \ + "$root/docs/install/examples/workspace-bindings.env.example" <<'PY' +import pathlib +import re +import sys +import yaml + +local = yaml.safe_load(pathlib.Path(sys.argv[1]).read_text()) +server = yaml.safe_load(pathlib.Path(sys.argv[2]).read_text()) +bindings = pathlib.Path(sys.argv[3]).read_text() +if local.get("profile") != "local" or server.get("profile") != "server": + raise SystemExit("installation examples must retain their local/server profiles") +for document in (local, server): + if "metadataGeneration" not in document or "authentication" not in document: + raise SystemExit("installation example lacks metadata or authentication configuration") +if re.search(r"(?:KEY|PASSWORD|TOKEN|SECRET)=[^\n#<][^\n]*", bindings): + raise SystemExit("workspace bindings example contains a secret value") +PY + echo "Installation examples contract passed" +} + +verify_all() { + verify_compose_topology + verify_navigation + verify_install_and_workspace_guides + verify_examples +} + +case "${1:-}" in --fixtures-only) - [[ $# -eq 1 ]] || { echo "usage: $0 --fixtures-only" >&2; exit 2; } - verify_internal_semantic_infrastructure_docs - echo "internal semantic infrastructure documentation contract passed" - verify_read_only_workspace_runtime_contract - verify_workspace_evidence_contract - verify_local_guide - verify_windows_line_endings_guide - verify_pi_management_guide - verify_server_guide - verify_reverse_proxy_nginx_guide - verify_reverse_proxy_caddy_guide - verify_local_installation_example - verify_server_installation_example - verify_manual local - verify_manual server - verify_compose_fixtures + [[ $# -eq 1 ]] || fail 'usage: verify-workspace-install-docs.sh --fixtures-only | --profile {local|server}' + verify_all ;; --profile) - profile="${2:-}" - [[ $# -eq 2 && "$profile" =~ ^(local|server)$ ]] \ - || { echo "usage: $0 --profile {local|server}" >&2; exit 2; } - if [[ "$profile" == local ]]; then - verify_internal_semantic_infrastructure_docs - verify_read_only_workspace_runtime_contract - verify_workspace_evidence_contract - verify_local_guide - verify_windows_line_endings_guide - verify_pi_management_guide - verify_local_installation_example - else - verify_internal_semantic_infrastructure_docs - verify_read_only_workspace_runtime_contract - verify_workspace_evidence_contract - verify_server_guide - verify_reverse_proxy_nginx_guide - verify_reverse_proxy_caddy_guide - verify_caddy_effective_proxy_guide - "$root/scripts/test-server-operator-permissions.sh" - verify_server_installation_example - fi - verify_manual "$profile" - verify_compose_fixtures - echo "== Run isolated workspace-registry bootstrap and recovery smoke ==" - ( - cd "$root" - env -u WORKSPACE_GIT_REMOTE ./scripts/workspace-registry-smoke.sh - ) - echo "$profile installation documentation verification passed" + [[ $# -eq 2 && "${2:-}" =~ ^(local|server)$ ]] \ + || fail 'usage: verify-workspace-install-docs.sh --fixtures-only | --profile {local|server}' + verify_all + echo "$2 installation documentation verification passed" ;; *) - echo "usage: $0 --fixtures-only | --profile {local|server}" >&2 - exit 2 + fail 'usage: verify-workspace-install-docs.sh --fixtures-only | --profile {local|server}' ;; esac