1505 lines
66 KiB
Bash
Executable File
1505 lines
66 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Verify canonical local/server installation manuals and their base+override Compose paths.
|
|
set -euo pipefail
|
|
|
|
root="$(cd "$(dirname "$0")/.." && pwd -P)"
|
|
mode="${1:-}"
|
|
|
|
trim() {
|
|
local value="$1"
|
|
value="${value#"${value%%[![:space:]]*}"}"
|
|
value="${value%"${value##*[![:space:]]}"}"
|
|
printf '%s' "$value"
|
|
}
|
|
|
|
is_safe_absolute_path() {
|
|
local value="$1" segment
|
|
local -a segments
|
|
[[ "$value" == /* && "$value" != *//* ]] || return 1
|
|
IFS=/ read -r -a segments <<<"$value"
|
|
for segment in "${segments[@]}"; do
|
|
[[ "$segment" != . && "$segment" != .. ]] || return 1
|
|
done
|
|
}
|
|
|
|
verify_path_variable_values() {
|
|
local source="$1" line trimmed name value
|
|
while IFS= read -r line || [[ -n "$line" ]]; do
|
|
trimmed="$(trim "$line")"
|
|
if [[ "$trimmed" == *=* || "$trimmed" == *:* ]]; then
|
|
name="$(trim "${trimmed%%[=:]*}")"
|
|
value="$(trim "${trimmed#"$name"}")"
|
|
value="$(trim "${value#[:=]}")"
|
|
if [[ "$name" =~ ^[A-Za-z_][A-Za-z0-9_]*_(FILE|SOURCE)$ ]]; then
|
|
value="$(trim "${value%%#*}")"
|
|
value="${value#\"}"; value="${value%\"}"
|
|
value="${value#\'}"; value="${value%\'}"
|
|
if [[ -n "$value" ]] && ! is_safe_absolute_path "$value"; then
|
|
echo "unsafe path value for $name in $source" >&2
|
|
return 1
|
|
fi
|
|
fi
|
|
fi
|
|
done <"$source"
|
|
}
|
|
|
|
require_headings() {
|
|
local source="$1" label="$2"
|
|
shift 2
|
|
local heading
|
|
for heading in "$@"; do
|
|
grep -Fqx "## $heading" "$source" || {
|
|
echo "missing required heading in $label: $heading" >&2
|
|
return 1
|
|
}
|
|
done
|
|
}
|
|
|
|
require_text() {
|
|
local source="$1" label="$2"
|
|
shift 2
|
|
local expected
|
|
for expected in "$@"; do
|
|
grep -Fq -- "$expected" "$source" || {
|
|
echo "$label lacks required instruction: $expected" >&2
|
|
return 1
|
|
}
|
|
done
|
|
}
|
|
|
|
verify_local_guide() {
|
|
local guide="$root/docs/install/local.md"
|
|
[[ -f "$guide" ]] || {
|
|
echo "missing local installation guide: docs/install/local.md" >&2
|
|
return 1
|
|
}
|
|
require_headings "$guide" "local installation guide" \
|
|
"Choose your platform" \
|
|
"Prerequisites" \
|
|
"Clone and verify LF" \
|
|
"Create the local operator files" \
|
|
"Address external services" \
|
|
"Build ThothII and thothctl" \
|
|
"Start and verify" \
|
|
"Update an installation" \
|
|
"Back up and restore" \
|
|
"Data-preserving uninstall" \
|
|
"Next: workspaces and Pi"
|
|
require_text "$guide" "local installation guide" \
|
|
"git clone" \
|
|
"bash scripts/verify-line-endings.sh" \
|
|
"deploy/env/local.env" \
|
|
"host.docker.internal" \
|
|
"host-gateway" \
|
|
"container 127.0.0.1" \
|
|
"bash scripts/build-local.sh" \
|
|
"scripts/build-local.ps1" \
|
|
"bash scripts/build-thothctl.sh" \
|
|
"thothctl --installation" \
|
|
"curl --fail http://127.0.0.1:8080/health" \
|
|
"http://127.0.0.1:8080" \
|
|
"git pull --ff-only" \
|
|
"docker compose down --volumes"
|
|
node - "$guide" <<'NODE'
|
|
const fs = require("fs");
|
|
const source = fs.readFileSync(process.argv[2], "utf8");
|
|
|
|
function section(name) {
|
|
const marker = `## ${name}`;
|
|
const start = source.indexOf(marker);
|
|
if (start < 0) throw new Error(`missing section: ${name}`);
|
|
const next = source.indexOf("\n## ", start + marker.length);
|
|
return source.slice(start, next < 0 ? source.length : next);
|
|
}
|
|
|
|
function blocks(name, language) {
|
|
const expression = new RegExp("```" + language + "\\n([\\s\\S]*?)```", "g");
|
|
return [...section(name).matchAll(expression)].map((match) => match[1]);
|
|
}
|
|
|
|
function requireTokens(label, text, tokens) {
|
|
for (const token of tokens) {
|
|
if (!text.includes(token)) throw new Error(`${label} lacks structural token: ${token}`);
|
|
}
|
|
}
|
|
|
|
function requirePattern(label, text, pattern) {
|
|
if (!pattern.test(text)) throw new Error(label);
|
|
}
|
|
|
|
let inCodeFence = false;
|
|
for (const line of source.split(/\n/)) {
|
|
if (line.trimStart().startsWith("```")) {
|
|
inCodeFence = !inCodeFence;
|
|
continue;
|
|
}
|
|
if (!line.includes("docker compose down --volumes")) continue;
|
|
const normalized = line.toLowerCase().replaceAll("*", "");
|
|
if (inCodeFence || !/(do not|never)/.test(normalized) || /^\s*(docker|&?\s*docker)/.test(normalized)) {
|
|
throw new Error("docker compose down --volumes must appear only in an explicit prose prohibition");
|
|
}
|
|
}
|
|
|
|
const setupPowerShell = blocks("Create the local operator files", "powershell").join("\n");
|
|
requireTokens("native PowerShell setup", setupPowerShell, [
|
|
"Copy-Item", "New-Item", "icacls.exe", "/inheritance:r", "/grant:r",
|
|
"WindowsIdentity", "deploy/env/local.env.example", "thothii-installation.yaml",
|
|
]);
|
|
|
|
const healthPowerShell = blocks("Start and verify", "powershell").join("\n");
|
|
requireTokens("native PowerShell health", healthPowerShell, [
|
|
"curl.exe --fail", "http://127.0.0.1:8080/health", "http://127.0.0.1:8787/health",
|
|
"pi doctor", "pi test",
|
|
]);
|
|
|
|
const updateShell = blocks("Update an installation", "sh").join("\n");
|
|
requireTokens("installation-aware source update", updateShell, [
|
|
"NEXT_PI_VERSION", "RUNNING_PI_VERSION", "--source build", "git rev-parse HEAD",
|
|
"pi status", "status", "doctor", "curl --fail", "set -euo pipefail",
|
|
"git status --porcelain --untracked-files=all", "USES_BASE_CORE", "thothii-core:local",
|
|
]);
|
|
if (/\|\|\s*true|;\s*true\b/.test(updateShell)) throw new Error("POSIX source update contains a failure-bypass command");
|
|
requirePattern("POSIX source update does not fail closed: source pull", updateShell,
|
|
/if ! git pull --ff-only; then abort_update/);
|
|
requirePattern("POSIX source update does not fail closed: installation status", updateShell,
|
|
/if ! INSTALLATION_STATUS="\$\("\$THTCTL" --installation "\$INSTALLATION" status\)"; then/);
|
|
requirePattern("POSIX source update does not fail closed: Pi status", updateShell,
|
|
/if ! RUNNING_PI_VERSION="\$\("\$THTCTL" --installation "\$INSTALLATION" pi status\)"; then/);
|
|
requirePattern("POSIX source update does not fail closed: local build", updateShell,
|
|
/if ! bash scripts\/build-local\.sh; then/);
|
|
requirePattern("POSIX source update does not fail closed: thothctl build", updateShell,
|
|
/if ! bash scripts\/build-thothctl\.sh; then/);
|
|
requirePattern("POSIX source update lacks the same-version/no-selector path", updateShell,
|
|
/if \[\[ "\$NEXT_PI_VERSION" == "\$RUNNING_PI_VERSION" \]\]; then[\s\S]*"\$USES_BASE_CORE" == true[\s\S]*TRANSACTIONAL_PI_UPDATE=false/);
|
|
for (const [label, pattern] of [
|
|
["installation start", /if ! "\$THTCTL" --installation "\$INSTALLATION" start; then/],
|
|
["frontend health", /if ! curl --fail http:\/\/127\.0\.0\.1:8080\/health; then/],
|
|
["core health", /if ! curl --fail http:\/\/127\.0\.0\.1:8787\/health; then/],
|
|
["final status", /if ! FINAL_STATUS="\$\("\$THTCTL" --installation "\$INSTALLATION" status\)"; then/],
|
|
["final Pi status", /if ! FINAL_PI_STATUS="\$\("\$THTCTL" --installation "\$INSTALLATION" pi status\)"; then/],
|
|
["final doctor", /if ! "\$THTCTL" --installation "\$INSTALLATION" doctor; then/],
|
|
]) requirePattern(`POSIX source update does not fail closed: ${label}`, updateShell, pattern);
|
|
const provenance = updateShell.indexOf("printf 'Built source revision:");
|
|
if (provenance < updateShell.lastIndexOf("require_clean_source") ||
|
|
provenance < updateShell.indexOf('abort_update "final doctor failed"')) {
|
|
throw new Error("POSIX source revision provenance is printed before final checks");
|
|
}
|
|
|
|
const updatePowerShell = blocks("Update an installation", "powershell").join("\n");
|
|
requireTokens("native PowerShell source update", updatePowerShell, [
|
|
"$NextPiVersion", "$RunningPiVersion", "--source build", "git rev-parse HEAD",
|
|
"pi status", "status", "doctor", "curl.exe --fail", "throw", "$ErrorActionPreference = 'Stop'",
|
|
"git status --porcelain --untracked-files=all", "$UsesBaseCore", "thothii-core:local",
|
|
"$TransactionalPiUpdate = $false",
|
|
]);
|
|
for (const [command, step] of [
|
|
["git pull --ff-only", "source pull"],
|
|
["$InstallationStatus = @(& $THTCTL --installation $INSTALLATION status)", "installation status"],
|
|
["$RunningPiStatus = (& $THTCTL --installation $INSTALLATION pi status)", "Pi status"],
|
|
["powershell -ExecutionPolicy Bypass -File scripts/build-local.ps1", "local image build"],
|
|
["& \"C:\\Program Files\\Git\\bin\\bash.exe\" scripts/build-thothctl.sh", "thothctl build"],
|
|
["curl.exe --fail --silent --show-error http://127.0.0.1:8080/health", "frontend health check"],
|
|
["curl.exe --fail --silent --show-error http://127.0.0.1:8787/health", "core health check"],
|
|
["$FinalPiStatus = (& $THTCTL --installation $INSTALLATION pi status)", "final Pi status"],
|
|
["& $THTCTL --installation $INSTALLATION doctor", "final doctor"],
|
|
]) {
|
|
const commandAt = updatePowerShell.indexOf(command);
|
|
const checkAt = updatePowerShell.indexOf(`Assert-NativeSuccess '${step}'`, commandAt);
|
|
if (commandAt < 0 || checkAt < commandAt || checkAt - commandAt > 220) {
|
|
throw new Error(`PowerShell source update does not propagate failure: ${step}`);
|
|
}
|
|
}
|
|
requirePattern("PowerShell source update lacks the same-version/no-selector path", updatePowerShell,
|
|
/if \(\$NextPiVersion -eq \$RunningPiVersion\) \{[\s\S]*-not \$UsesBaseCore[\s\S]*\$TransactionalPiUpdate = \$false/);
|
|
|
|
const backupPowerShell = blocks("Back up and restore", "powershell").join("\n");
|
|
requireTokens("native PowerShell backup/restore", backupPowerShell, [
|
|
"$BackupDir", "$Volume", "-czf", "$TargetVolume", "$Archive", "Split-Path -Parent",
|
|
"Split-Path -Leaf", "test -z", "-xzf",
|
|
]);
|
|
|
|
for (const block of [setupPowerShell, healthPowerShell, updatePowerShell, backupPowerShell]) {
|
|
if (/\$\((dirname|basename)\b|\bmkdir -p\b|\bchmod\s+[0-7]/.test(block)) {
|
|
throw new Error("native PowerShell block contains a POSIX-only command sequence");
|
|
}
|
|
}
|
|
NODE
|
|
local update_fixture update_script fake_bin calls output status
|
|
update_fixture="$(mktemp -d "${TMPDIR%/}/thoth-source-update.XXXXXX")"
|
|
trap 'rm -rf "$update_fixture"' RETURN
|
|
update_script="$update_fixture/update.sh"
|
|
awk '
|
|
/^## Update an installation$/ { in_section=1; next }
|
|
in_section && /^```sh$/ { in_code=1; next }
|
|
in_code && /^```$/ { exit }
|
|
in_code { print }
|
|
' "$guide" >"$update_script"
|
|
chmod 0700 "$update_script"
|
|
mkdir -p "$update_fixture/project/docker" "$update_fixture/project/scripts" "$update_fixture/bin"
|
|
printf 'ARG PI_VERSION=0.80.3\n' >"$update_fixture/project/docker/core.Dockerfile"
|
|
printf '%s\n' \
|
|
'#!/bin/sh' \
|
|
'printf "git %s\n" "$*" >>"$CALLS"' \
|
|
'case "$1" in' \
|
|
' status) if [ "$FAIL_STEP" = dirty ]; then printf "?? untracked-build-context\n"; fi ;;' \
|
|
' pull) [ "$FAIL_STEP" != pull ] || exit 9 ;;' \
|
|
' rev-parse) printf "0123456789abcdef\n" ;;' \
|
|
'esac' \
|
|
'exit 0' >"$update_fixture/bin/git"
|
|
printf '%s\n' \
|
|
'#!/bin/sh' \
|
|
'printf "bash %s\n" "$*" >>"$CALLS"' \
|
|
'if [ "$1" = scripts/build-local.sh ] && [ "$FAIL_STEP" = build ]; then exit 8; fi' \
|
|
'exit 0' >"$update_fixture/bin/bash"
|
|
printf '%s\n' \
|
|
'#!/bin/sh' \
|
|
'printf "thothctl %s\n" "$*" >>"$CALLS"' \
|
|
'case " $* " in' \
|
|
' *" pi status "*) [ "$FAIL_STEP" != status ] || exit 7; printf "Pi version: 0.80.3\n" ;;' \
|
|
' *" status "*) printf "[{\"Service\":\"core\",\"Image\":\"thothii-core:local\"}]\n" ;;' \
|
|
'esac' \
|
|
'exit 0' >"$update_fixture/bin/thothctl"
|
|
printf '%s\n' \
|
|
'#!/bin/sh' \
|
|
'printf "curl %s\n" "$*" >>"$CALLS"' \
|
|
'exit 0' >"$update_fixture/bin/curl"
|
|
chmod 0700 "$update_fixture/bin/git" "$update_fixture/bin/bash" \
|
|
"$update_fixture/bin/thothctl" "$update_fixture/bin/curl"
|
|
|
|
for fixture_step in clean dirty pull status build; do
|
|
calls="$update_fixture/calls-$fixture_step"
|
|
output="$update_fixture/output-$fixture_step"
|
|
: >"$calls"
|
|
set +e
|
|
(
|
|
cd "$update_fixture/project"
|
|
env PATH="$update_fixture/bin:$PATH" CALLS="$calls" FAIL_STEP="$fixture_step" \
|
|
THTCTL="$update_fixture/bin/thothctl" INSTALLATION="$update_fixture/installation.yaml" \
|
|
/bin/bash "$update_script"
|
|
) >"$output" 2>&1
|
|
status=$?
|
|
set -e
|
|
if [[ "$fixture_step" == clean ]]; then
|
|
[[ $status -eq 0 ]] || { echo "same-version/no-selector source fixture failed" >&2; return 1; }
|
|
grep -Fq 'Built source revision: 0123456789abcdef' "$output" || {
|
|
echo "successful source fixture did not report revision provenance" >&2; return 1;
|
|
}
|
|
if grep -Fq ' pi update ' "$calls"; then
|
|
echo "same-version/no-selector source fixture incorrectly invoked pi update" >&2
|
|
return 1
|
|
fi
|
|
grep -Fq 'bash scripts/build-local.sh' "$calls" || return 1
|
|
grep -Fq 'thothctl --installation ' "$calls" || return 1
|
|
else
|
|
[[ $status -ne 0 ]] || { echo "$fixture_step source failure fixture was accepted" >&2; return 1; }
|
|
if grep -Fq 'Built source revision:' "$output"; then
|
|
echo "$fixture_step source failure fixture claimed revision provenance" >&2
|
|
return 1
|
|
fi
|
|
fi
|
|
done
|
|
echo "source update fail-closed semantics passed"
|
|
echo "local installation guide contract passed"
|
|
}
|
|
|
|
verify_windows_line_endings_guide() {
|
|
local guide="$root/docs/install/windows-line-endings.md"
|
|
[[ -f "$guide" ]] || {
|
|
echo "missing Windows line-ending guide: docs/install/windows-line-endings.md" >&2
|
|
return 1
|
|
}
|
|
require_headings "$guide" "Windows line-ending guide" \
|
|
"Recommended WSL2 clone" \
|
|
"Repository-local LF policy" \
|
|
"Verify after clone or pull" \
|
|
"Recover an existing CRLF clone"
|
|
require_text "$guide" "Windows line-ending guide" \
|
|
"git config --local core.autocrlf false" \
|
|
"bash scripts/verify-line-endings.sh" \
|
|
"git add --renormalize ." \
|
|
"git checkout-index --all --force" \
|
|
"git diff --cached --check" \
|
|
"reclone"
|
|
node - "$guide" <<'NODE'
|
|
const fs = require("fs");
|
|
const source = fs.readFileSync(process.argv[2], "utf8");
|
|
const lines = source.split(/\n/);
|
|
const sectionStart = source.indexOf("## Recover an existing CRLF clone");
|
|
const recovery = source.slice(sectionStart);
|
|
const shell = [...recovery.matchAll(/```sh\n([\s\S]*?)```/g)].map((match) => match[1]).join("\n");
|
|
const powershell = [...recovery.matchAll(/```powershell\n([\s\S]*?)```/g)].map((match) => match[1]).join("\n");
|
|
const commands = [
|
|
"git add --renormalize .",
|
|
"git checkout-index --all --force --prefix=",
|
|
"bash scripts/verify-line-endings.sh",
|
|
];
|
|
let prior = -1;
|
|
for (const command of commands) {
|
|
const index = lines.findIndex((line, candidate) => candidate > prior && line.trim().includes(command));
|
|
if (index < 0) throw new Error(`CRLF recovery lacks ordered command: ${command}`);
|
|
prior = index;
|
|
}
|
|
for (const token of [
|
|
"set -euo pipefail", "validate_index_export", "validate_worktree_modes", "rewrite_index_entry", "git ls-files -s -z",
|
|
"100644", "100755", "120000", "readlink", "ln -s", "if ! git checkout-index",
|
|
]) {
|
|
if (!shell.includes(token)) throw new Error(`POSIX CRLF repair lacks fail-closed semantic: ${token}`);
|
|
}
|
|
if (/\|\|\s*true|;\s*true\b/.test(shell)) throw new Error("POSIX CRLF repair contains a failure-bypass command");
|
|
const exportAt = shell.indexOf("if ! git checkout-index");
|
|
const validationAt = shell.indexOf("if ! validate_index_export", exportAt);
|
|
const exportedBytesAt = shell.indexOf('if ! bash scripts/verify-line-endings.sh "$REPAIR_DIR"', validationAt);
|
|
const rewriteAt = shell.indexOf("if ! git ls-files -s -z", exportedBytesAt);
|
|
const finalModesAt = shell.indexOf("if ! validate_worktree_modes; then", rewriteAt);
|
|
const finalAt = shell.indexOf("if ! bash scripts/verify-line-endings.sh; then", finalModesAt);
|
|
if ([exportAt, validationAt, exportedBytesAt, rewriteAt, finalModesAt, finalAt].some((index) => index < 0) ||
|
|
!(exportAt < validationAt && validationAt < exportedBytesAt && exportedBytesAt < rewriteAt && rewriteAt < finalModesAt && finalModesAt < finalAt)) {
|
|
throw new Error("POSIX CRLF repair does not prove a complete export before destructive rewrite");
|
|
}
|
|
for (const token of [
|
|
"$ErrorActionPreference = 'Stop'", "Assert-NativeSuccess 'index renormalization'",
|
|
"Assert-NativeSuccess 'normalized index check'", "Assert-NativeSuccess 'index export'",
|
|
"Assert-NativeSuccess 'index inventory'", "100644", "100755", "120000", "SymbolicLink",
|
|
"-ErrorAction Stop", "$WorktreeItem", "Assert-NativeSuccess 'repaired worktree LF verification'",
|
|
]) {
|
|
if (!powershell.includes(token)) throw new Error(`PowerShell CRLF repair lacks failure propagation: ${token}`);
|
|
}
|
|
const warningPattern = /WARNING[^\n]*destructive[^\n]*(backup|commit)/i;
|
|
const powerShellWarningAt = powershell.indexOf("# WARNING: destructive copy");
|
|
const powerShellRewriteAt = powershell.indexOf("foreach ($Entry in $IndexEntries)", powerShellWarningAt);
|
|
if (!warningPattern.test(shell.slice(Math.max(0, rewriteAt - 180), rewriteAt)) ||
|
|
powerShellRewriteAt < 0 ||
|
|
!warningPattern.test(powershell.slice(Math.max(0, powerShellRewriteAt - 180), powerShellRewriteAt))) {
|
|
throw new Error("worktree rewrite lacks an immediate destructive warning requiring backup/commit");
|
|
}
|
|
NODE
|
|
local repair_root repair_script real_git partial_repo clean_repo partial_output repair_status
|
|
repair_root="$(mktemp -d "${TMPDIR%/}/thoth-crlf-repair.XXXXXX")"
|
|
trap 'rm -rf "$repair_root"' RETURN
|
|
repair_script="$repair_root/repair.sh"
|
|
awk '
|
|
/^## Recover an existing CRLF clone$/ { in_section=1; next }
|
|
in_section && /^```sh$/ { in_code=1; next }
|
|
in_code && /^```$/ { exit }
|
|
in_code { print }
|
|
' "$guide" >"$repair_script"
|
|
chmod 0700 "$repair_script"
|
|
|
|
prepare_crlf_fixture() {
|
|
local repository="$1"
|
|
mkdir -p "$repository/scripts"
|
|
git -C "$repository" init -q
|
|
printf '*.sh text eol=lf\n' >"$repository/.gitattributes"
|
|
printf '#!/bin/sh\nexit 0\n' >"$repository/repair.sh"
|
|
printf 'target\n' >"$repository/target.txt"
|
|
cp "$root/scripts/verify-line-endings.sh" "$repository/scripts/verify-line-endings.sh"
|
|
ln -s target.txt "$repository/workspace-link"
|
|
git -C "$repository" add .gitattributes repair.sh target.txt workspace-link \
|
|
scripts/verify-line-endings.sh 2>/dev/null
|
|
printf '#!/bin/sh\r\nexit 0\r\n' >"$repository/repair.sh"
|
|
}
|
|
|
|
partial_repo="$repair_root/partial/worktree"
|
|
mkdir -p "$partial_repo" "$repair_root/partial/bin"
|
|
prepare_crlf_fixture "$partial_repo"
|
|
real_git="$(command -v git)"
|
|
printf '%s\n' \
|
|
'#!/bin/sh' \
|
|
'"$REAL_GIT" "$@"' \
|
|
'status=$?' \
|
|
'if [ $status -eq 0 ] && [ "$1" = checkout-index ]; then rm -f "$PARTIAL_EXPORT_PATH"; fi' \
|
|
'exit $status' >"$repair_root/partial/bin/git"
|
|
chmod 0700 "$repair_root/partial/bin/git"
|
|
partial_output="$repair_root/partial/output"
|
|
set +e
|
|
(
|
|
cd "$partial_repo"
|
|
env PATH="$repair_root/partial/bin:$PATH" REAL_GIT="$real_git" \
|
|
PARTIAL_EXPORT_PATH="$repair_root/partial/ThothII-lf-repair/repair.sh" \
|
|
/bin/bash "$repair_script"
|
|
) >"$partial_output" 2>&1
|
|
repair_status=$?
|
|
set -e
|
|
[[ $repair_status -ne 0 ]] || { echo "partial CRLF export fixture was accepted" >&2; return 1; }
|
|
LC_ALL=C grep -q $'\r' "$partial_repo/repair.sh" || {
|
|
echo "partial CRLF export fixture rewrote bytes before complete validation" >&2; return 1;
|
|
}
|
|
[[ -L "$partial_repo/workspace-link" && "$(readlink "$partial_repo/workspace-link")" == target.txt ]] || {
|
|
echo "partial CRLF export fixture changed the tracked symlink" >&2; return 1;
|
|
}
|
|
|
|
clean_repo="$repair_root/clean/worktree"
|
|
mkdir -p "$clean_repo"
|
|
prepare_crlf_fixture "$clean_repo"
|
|
(cd "$clean_repo" && /bin/bash "$repair_script") >/dev/null
|
|
"$root/scripts/verify-line-endings.sh" "$clean_repo"
|
|
[[ -L "$clean_repo/workspace-link" && "$(readlink "$clean_repo/workspace-link")" == target.txt ]] || {
|
|
echo "successful CRLF repair did not preserve the mode-120000 symlink" >&2; return 1;
|
|
}
|
|
echo "CRLF recovery rewrites bytes and preserves mode-120000 symlinks passed"
|
|
echo "Windows line-ending recovery guide contract passed"
|
|
}
|
|
|
|
verify_pi_management_guide() {
|
|
local guide="$root/docs/install/pi-management.md"
|
|
[[ -f "$guide" ]] || {
|
|
echo "missing Pi management guide: docs/install/pi-management.md" >&2
|
|
return 1
|
|
}
|
|
require_headings "$guide" "Pi management guide" \
|
|
"Who can use Pi Management" \
|
|
"Use the Pi Management page" \
|
|
"Use thothctl" \
|
|
"Handle credentials and secrets" \
|
|
"Update and roll back Pi" \
|
|
"Recover a failed update" \
|
|
"Direct support access"
|
|
require_text "$guide" "Pi management guide" \
|
|
"pi status" \
|
|
"pi doctor" \
|
|
"pi test" \
|
|
"pi check" \
|
|
"pi configure" \
|
|
"pi update" \
|
|
"pi rollback --yes" \
|
|
"pi maintenance status" \
|
|
"pi maintenance recover --yes" \
|
|
"pi logs" \
|
|
"/run/secrets" \
|
|
"Raw Compose access is unsupported" \
|
|
"no browser shell" \
|
|
"does not mount the Docker socket"
|
|
node - "$guide" <<'NODE'
|
|
const fs = require("fs");
|
|
const source = fs.readFileSync(process.argv[2], "utf8");
|
|
if (/docker\s+compose(?:.|\n){0,160}\bexec\b(?:.|\n){0,80}\bcore\b(?:.|\n){0,80}\bpi\b/i.test(source)) {
|
|
throw new Error("raw non-installation-aware Compose Pi access is forbidden");
|
|
}
|
|
const marker = "## Direct support access";
|
|
const start = source.indexOf(marker);
|
|
const support = start < 0 ? "" : source.slice(start, source.indexOf("\n## ", start + marker.length) < 0
|
|
? source.length : source.indexOf("\n## ", start + marker.length));
|
|
for (const token of ["unsupported", "thothctl", "pi status", "pi doctor", "pi test", "pi logs"]) {
|
|
if (!support.toLowerCase().includes(token.toLowerCase())) {
|
|
throw new Error(`direct support section lacks installation-aware diagnostic: ${token}`);
|
|
}
|
|
}
|
|
NODE
|
|
echo "Pi management guide contract passed"
|
|
}
|
|
|
|
verify_server_guide() {
|
|
local guide="$root/docs/install/server.md"
|
|
[[ -f "$guide" ]] || {
|
|
echo "missing server installation guide sections: docs/install/server.md" >&2
|
|
return 1
|
|
}
|
|
require_headings "$guide" "server installation guide" \
|
|
"Deployment contract" \
|
|
"Service account and directories" \
|
|
"Firewall and network boundaries" \
|
|
"Address co-resident external services" \
|
|
"Prepare operator files and secrets" \
|
|
"Build locally or select pinned images" \
|
|
"Install thothctl" \
|
|
"Start and verify readiness" \
|
|
"Configure TLS and upstream authentication" \
|
|
"Operate Pi, drain, and roll back" \
|
|
"Back up and restore" \
|
|
"Diagnostics" \
|
|
"Data-preserving uninstall"
|
|
require_text "$guide" "server installation guide" \
|
|
"frontend" \
|
|
"core" \
|
|
"UID/GID 10001" \
|
|
"thothii-ops" \
|
|
"-m 2770 /srv/thothii/operator" \
|
|
"chmod 0660 /srv/thothii/operator/server.env" \
|
|
"THT_THOTHCTL_OUTPUT_DIRECTORY=/srv/thothii/operator/build-output" \
|
|
"/srv/thothii" \
|
|
"example operator root" \
|
|
"/run/secrets" \
|
|
"Git-backed workspace registry is the source of truth" \
|
|
"host.docker.internal" \
|
|
"host-gateway" \
|
|
"com.docker.network.bridge.name" \
|
|
"DOCKER-USER" \
|
|
"iptables -I INPUT" \
|
|
"container 127.0.0.1" \
|
|
"collection" \
|
|
"embedding" \
|
|
"bash scripts/build-local.sh" \
|
|
"@sha256:" \
|
|
"bash scripts/build-thothctl.sh" \
|
|
"thothctl --installation" \
|
|
"sessions migrate --yes" \
|
|
'"pending":[]' \
|
|
'"drifted":[]' \
|
|
"remove --yes" \
|
|
"THT_BACKUP_ROOT=/srv/thothii-backups" \
|
|
"sha256sum --check SHA256SUMS" \
|
|
"curl --fail http://127.0.0.1:8080/health" \
|
|
"https://thoth.example.com" \
|
|
"pi update" \
|
|
"--drain" \
|
|
"pi rollback --yes" \
|
|
"pi maintenance recover --yes" \
|
|
"docker compose down --volumes" \
|
|
"reverse-proxy-nginx.md" \
|
|
"reverse-proxy-caddy.md"
|
|
if ! grep -Eq '^sudo install -d -o 10001 -g thothii-ops -m 2750 /srv/thothii$' "$guide"; then
|
|
echo "server installation guide does not set parent traversal boundary" >&2
|
|
return 1
|
|
fi
|
|
node - "$guide" <<'NODE'
|
|
const fs = require("fs");
|
|
const source = fs.readFileSync(process.argv[2], "utf8");
|
|
if (/omics_portal|chirone|localllm_default|datamart-builder|compose\.production|compose\.psd-local/i.test(source)) {
|
|
throw new Error("server installation guide introduces forbidden application coupling");
|
|
}
|
|
if (/\/var\/run\/docker\.sock|docker\.sock/i.test(source)) {
|
|
throw new Error("server installation guide introduces a Docker socket dependency");
|
|
}
|
|
for (const line of source.split(/\n/)) {
|
|
const match = line.match(/^\s*([A-Z][A-Z0-9_]*(?:PASSWORD|TOKEN|API_KEY|SECRET)[A-Z0-9_]*)\s*=\s*(\S.*)$/);
|
|
if (!match) continue;
|
|
const [, name, rawValue] = match;
|
|
const value = rawValue.trim();
|
|
if (!/(?:_FILE|_SOURCE)$/.test(name) && value && !/^\$\{?[A-Z_][A-Z0-9_]*\}?$/.test(value)) {
|
|
throw new Error("server installation guide embeds a secret value");
|
|
}
|
|
}
|
|
let inCodeFence = false;
|
|
for (const line of source.split(/\n/)) {
|
|
if (line.trimStart().startsWith("```")) {
|
|
inCodeFence = !inCodeFence;
|
|
continue;
|
|
}
|
|
if (!line.includes("docker compose down --volumes")) continue;
|
|
const normalized = line.toLowerCase().replaceAll("*", "");
|
|
if (inCodeFence || !/(do not|never)/.test(normalized) || /^\s*(docker|&?\s*docker)/.test(normalized)) {
|
|
throw new Error("server docker compose down --volumes must appear only in an explicit prose prohibition");
|
|
}
|
|
}
|
|
if (/```(?:sh|bash)\n[\s\S]*?\bdocker\s+rm\b[\s\S]*?```/i.test(source)) {
|
|
throw new Error("server uninstall bypasses installation-aware removal");
|
|
}
|
|
if (/host-gateway[^\n]{0,120}(?:listen|listening|bound)[^\n]{0,80}127\.0\.0\.1|(?:listen|listening|bound)[^\n]{0,80}127\.0\.0\.1[^\n]{0,120}host-gateway/i.test(source)) {
|
|
throw new Error("server host-gateway guidance assumes a host loopback listener");
|
|
}
|
|
const pinnedStart = source.indexOf("## Build locally or select pinned images");
|
|
const pinnedEnd = source.indexOf("\n## ", pinnedStart + 3);
|
|
const pinnedSection = source.slice(pinnedStart, pinnedEnd < 0 ? source.length : pinnedEnd);
|
|
const pinnedBlock = [...pinnedSection.matchAll(/```yaml\n([\s\S]*?)```/g)].map((match) => match[1])
|
|
.find((block) => block.includes("session-migrate:")) || "";
|
|
function pinnedService(name) {
|
|
const match = pinnedBlock.match(new RegExp(`^ ${name}:\\n((?: [^\\n]*\\n)+)`, "m"));
|
|
return match ? match[1] : "";
|
|
}
|
|
const pinnedCore = pinnedService("core");
|
|
const pinnedMigrator = pinnedService("session-migrate");
|
|
const pinnedFrontend = pinnedService("frontend");
|
|
const coreImage = pinnedCore.match(/image:\s*(\S+)/)?.[1];
|
|
const migratorImage = pinnedMigrator.match(/image:\s*(\S+)/)?.[1];
|
|
const frontendImage = pinnedFrontend.match(/image:\s*(\S+)/)?.[1];
|
|
if (![pinnedCore, pinnedMigrator, pinnedFrontend].every((block) => block.includes("build: !reset null")) ||
|
|
!coreImage || coreImage !== migratorImage || !/@sha256:<64-lowercase-hex-digits>$/.test(coreImage) ||
|
|
!frontendImage || !/@sha256:<64-lowercase-hex-digits>$/.test(frontendImage)) {
|
|
throw new Error("server pinned image override must pin core, session-migrate, and frontend without builds");
|
|
}
|
|
if (/session-migrate:[\s\S]{0,180}image:\s*thothii-core:local/.test(source) &&
|
|
/core:[\s\S]{0,180}image:\s*registry\.[^\n]+@sha256:[a-f0-9]{64}/.test(source)) {
|
|
throw new Error("server pinned migration image must equal the pinned core image");
|
|
}
|
|
if (/```(?:sh|bash)\n[\s\S]*?\bdocker compose\s+(?:up|stop|down|restart|pull|build)\b[\s\S]*?```/i.test(source)) {
|
|
throw new Error("server lifecycle must use thothctl, not raw Docker Compose");
|
|
}
|
|
NODE
|
|
echo "server installation guide contract passed"
|
|
}
|
|
|
|
verify_reverse_proxy_nginx_guide() {
|
|
local guide="$root/docs/install/reverse-proxy-nginx.md"
|
|
[[ -f "$guide" ]] || {
|
|
echo "missing Nginx reverse-proxy guide: docs/install/reverse-proxy-nginx.md" >&2
|
|
return 1
|
|
}
|
|
require_headings "$guide" "Nginx reverse-proxy guide" \
|
|
"Trust boundary" \
|
|
"Example configuration" \
|
|
"Validate and reload" \
|
|
"Test authentication and SSE"
|
|
require_text "$guide" "Nginx reverse-proxy guide" \
|
|
"Forwarding identity headers alone does not authenticate a user" \
|
|
"authentication gateway" \
|
|
"2xx" \
|
|
"TLS" \
|
|
"frontend"
|
|
node - "$guide" <<'NODE'
|
|
const fs = require("fs");
|
|
const source = fs.readFileSync(process.argv[2], "utf8");
|
|
const block = [...source.matchAll(/```nginx\n([\s\S]*?)```/g)].map((match) => match[1]).join("\n");
|
|
function stripNginxComments(text) {
|
|
let effective = "";
|
|
let quote = null;
|
|
let escaped = false;
|
|
let comment = false;
|
|
for (const character of text) {
|
|
if (comment) {
|
|
if (character === "\n") {
|
|
effective += character;
|
|
comment = false;
|
|
}
|
|
continue;
|
|
}
|
|
if (escaped) {
|
|
effective += character;
|
|
escaped = false;
|
|
continue;
|
|
}
|
|
if (character === "\\") {
|
|
effective += character;
|
|
escaped = true;
|
|
continue;
|
|
}
|
|
if (quote !== null) {
|
|
effective += character;
|
|
if (character === quote) quote = null;
|
|
continue;
|
|
}
|
|
if (character === '"' || character === "'") {
|
|
effective += character;
|
|
quote = character;
|
|
continue;
|
|
}
|
|
if (character === "#") {
|
|
comment = true;
|
|
continue;
|
|
}
|
|
effective += character;
|
|
}
|
|
return effective;
|
|
}
|
|
const effectiveBlock = stripNginxComments(block);
|
|
const tokens = [
|
|
"listen 443 ssl;", "ssl_certificate ", "ssl_certificate_key ",
|
|
"location = /_authenticate {", "internal;", "proxy_pass http://auth-gateway:4180/verify;",
|
|
"auth_request /_authenticate;", "proxy_pass http://127.0.0.1:8080;",
|
|
"proxy_http_version 1.1;", "proxy_buffering off;", "proxy_cache off;",
|
|
"proxy_read_timeout 3600s;",
|
|
];
|
|
if (/127\.0\.0\.1:8787|\bcore:8787\b/.test(effectiveBlock) || !effectiveBlock.includes("http://127.0.0.1:8080")) {
|
|
throw new Error("Nginx proxy must forward only to frontend on 127.0.0.1:8080");
|
|
}
|
|
for (const token of tokens) {
|
|
if (!effectiveBlock.includes(token)) throw new Error(`Nginx proxy lacks structural token: ${token}`);
|
|
}
|
|
if (/proxy_set_header\s+X-Thoth-Trusted-[^;]+\$http_/i.test(effectiveBlock)) {
|
|
throw new Error("Nginx proxy trusts a client-supplied identity header");
|
|
}
|
|
const identities = [
|
|
["issuer", "Principal-Issuer", "thoth_principal_issuer", "x_thoth_principal_issuer"],
|
|
["subject", "Principal-Subject", "thoth_principal_subject", "x_thoth_principal_subject"],
|
|
["display", "Principal-Display-Name", "thoth_principal_display_name", "x_thoth_principal_display_name"],
|
|
["admin", "Is-Admin", "thoth_is_admin", "x_thoth_is_admin"],
|
|
];
|
|
function escaped(value) { return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); }
|
|
function nginxLocations(text) {
|
|
const locations = [];
|
|
const pattern = /\blocation\s+([^\n{]+)\{/g;
|
|
for (const match of text.matchAll(pattern)) {
|
|
const opening = match.index + match[0].lastIndexOf("{");
|
|
let depth = 0;
|
|
let closing = -1;
|
|
for (let index = opening; index < text.length; index++) {
|
|
if (text[index] === "{") depth++;
|
|
if (text[index] === "}" && --depth === 0) {
|
|
closing = index;
|
|
break;
|
|
}
|
|
}
|
|
if (closing < 0) throw new Error(`Nginx proxy has unterminated location: ${match[1].trim()}`);
|
|
locations.push({selector: match[1].trim(), body: text.slice(opening + 1, closing)});
|
|
}
|
|
return locations;
|
|
}
|
|
const locations = nginxLocations(effectiveBlock);
|
|
const authLocations = locations.filter((location) => location.selector === "= /_authenticate");
|
|
if (authLocations.length !== 1) {
|
|
throw new Error("Nginx proxy must define exactly one authentication location");
|
|
}
|
|
const authLocation = authLocations[0].body;
|
|
const frontendLocations = locations.filter((location) =>
|
|
/proxy_pass\s+http:\/\/127\.0\.0\.1:8080\s*;/.test(location.body));
|
|
if (frontendLocations.length === 0) {
|
|
throw new Error("Nginx proxy lacks a frontend upstream location");
|
|
}
|
|
for (const location of locations) {
|
|
const upstreams = [...location.body.matchAll(/proxy_pass\s+([^;]+);/g)].map((match) => match[1].trim());
|
|
for (const upstream of upstreams) {
|
|
if (location.selector === "= /_authenticate" && upstream === "http://auth-gateway:4180/verify") continue;
|
|
if (upstream === "http://127.0.0.1:8080") continue;
|
|
throw new Error(`Nginx location proxies to an unreviewed upstream: ${upstream}`);
|
|
}
|
|
}
|
|
for (const frontendLocation of frontendLocations) {
|
|
if (!/auth_request\s+\/_authenticate\s*;/.test(frontendLocation.body)) {
|
|
throw new Error("Nginx frontend upstream location bypasses complete authentication contract");
|
|
}
|
|
}
|
|
for (const [label, publicName, variable, upstream] of identities) {
|
|
const trustedName = publicName === "Is-Admin" ? "Is-Admin" : publicName;
|
|
const publicClear = new RegExp(`proxy_set_header\\s+X-Thoth-${escaped(publicName)}\\s+"";`);
|
|
const trustedHeader = `X-Thoth-Trusted-${trustedName}`;
|
|
const trustedClear = new RegExp(`proxy_set_header\\s+${escaped(trustedHeader)}\\s+"";`);
|
|
const authPublicAt = authLocation.search(publicClear);
|
|
const authTrustedAt = authLocation.search(trustedClear);
|
|
if (authPublicAt < 0) {
|
|
throw new Error(`Nginx auth location does not clear inbound ${label} identity`);
|
|
}
|
|
if (authTrustedAt < 0) {
|
|
throw new Error(`Nginx auth location does not clear inbound trusted ${label} identity`);
|
|
}
|
|
for (const frontendLocation of frontendLocations) {
|
|
const frontendPublicAt = frontendLocation.body.search(publicClear);
|
|
if (frontendPublicAt < 0) {
|
|
throw new Error(`Nginx frontend location does not clear inbound ${label} identity`);
|
|
}
|
|
const normalizedFrontend = frontendLocation.body.replace(/\s+/g, " ");
|
|
const captureAt = normalizedFrontend.search(new RegExp(`auth_request_set\\s+\\$${variable}\\s+\\$upstream_http_${upstream};`));
|
|
if (captureAt < 0) {
|
|
throw new Error(`Nginx frontend location does not capture authenticated ${label} identity`);
|
|
}
|
|
const mapAt = normalizedFrontend.search(new RegExp(`proxy_set_header\\s+${escaped(trustedHeader)}\\s+\\$${variable};`));
|
|
if (mapAt < 0) {
|
|
throw new Error(`Nginx frontend location does not map authenticated ${label} identity`);
|
|
}
|
|
}
|
|
if (new RegExp(`auth_request_set\\s+\\$${variable}|proxy_set_header\\s+${escaped(trustedHeader)}\\s+\\$${variable};`).test(authLocation)) {
|
|
throw new Error(`Nginx auth location performs a forbidden ${label} capture or mapping`);
|
|
}
|
|
}
|
|
NODE
|
|
echo "Nginx reverse-proxy guide contract passed"
|
|
}
|
|
|
|
verify_reverse_proxy_caddy_guide() {
|
|
local guide="$root/docs/install/reverse-proxy-caddy.md"
|
|
[[ -f "$guide" ]] || {
|
|
echo "missing Caddy reverse-proxy guide: docs/install/reverse-proxy-caddy.md" >&2
|
|
return 1
|
|
}
|
|
require_headings "$guide" "Caddy reverse-proxy guide" \
|
|
"Trust boundary" \
|
|
"Example configuration" \
|
|
"Validate and reload" \
|
|
"Test authentication and SSE"
|
|
require_text "$guide" "Caddy reverse-proxy guide" \
|
|
"Forwarding identity headers alone does not authenticate a user" \
|
|
"authentication gateway" \
|
|
"2xx" \
|
|
"automatic HTTPS" \
|
|
"frontend"
|
|
node - "$guide" <<'NODE'
|
|
const fs = require("fs");
|
|
const source = fs.readFileSync(process.argv[2], "utf8");
|
|
const block = [...source.matchAll(/```caddyfile\n([\s\S]*?)```/g)].map((match) => match[1]).join("\n");
|
|
const tokens = [
|
|
"thoth.example.com {", "route {",
|
|
"forward_auth auth-gateway:4180 {", "uri /verify", "copy_headers {",
|
|
"reverse_proxy 127.0.0.1:8080 {", "flush_interval -1",
|
|
];
|
|
if (/127\.0\.0\.1:8787|\bcore:8787\b/.test(block) || !block.includes("127.0.0.1:8080")) {
|
|
throw new Error("Caddy proxy must forward only to frontend on 127.0.0.1:8080");
|
|
}
|
|
for (const token of tokens) {
|
|
if (!block.includes(token)) throw new Error(`Caddy proxy lacks structural token: ${token}`);
|
|
}
|
|
function directiveBlock(text, marker) {
|
|
const start = text.indexOf(marker);
|
|
if (start < 0) throw new Error(`Caddy proxy lacks scoped block: ${marker}`);
|
|
const opening = text.indexOf("{", start);
|
|
let depth = 0;
|
|
for (let index = opening; index < text.length; index++) {
|
|
if (text[index] === "{") depth++;
|
|
if (text[index] === "}" && --depth === 0) return {start, end: index, body: text.slice(opening + 1, index)};
|
|
}
|
|
throw new Error(`Caddy proxy has unterminated scoped block: ${marker}`);
|
|
}
|
|
const route = directiveBlock(block, "route {");
|
|
const forward = directiveBlock(route.body, "forward_auth auth-gateway:4180 {");
|
|
const forwardAt = route.body.indexOf("forward_auth auth-gateway:4180 {");
|
|
for (const [label, publicName, trustedName] of [
|
|
["issuer", "X-Thoth-Principal-Issuer", "X-Thoth-Trusted-Principal-Issuer"],
|
|
["subject", "X-Thoth-Principal-Subject", "X-Thoth-Trusted-Principal-Subject"],
|
|
["display", "X-Thoth-Principal-Display-Name", "X-Thoth-Trusted-Principal-Display-Name"],
|
|
["admin", "X-Thoth-Is-Admin", "X-Thoth-Trusted-Is-Admin"],
|
|
]) {
|
|
const publicClearAt = route.body.indexOf(`request_header -${publicName}`);
|
|
if (publicClearAt < 0) {
|
|
throw new Error(`Caddy proxy does not clear inbound ${label} identity`);
|
|
}
|
|
const trustedClearAt = route.body.indexOf(`request_header -${trustedName}`);
|
|
if (trustedClearAt < 0) {
|
|
throw new Error(`Caddy proxy does not clear inbound trusted ${label} identity`);
|
|
}
|
|
if (publicClearAt > forwardAt || trustedClearAt > forwardAt) {
|
|
throw new Error("Caddy identity clears must precede forward_auth");
|
|
}
|
|
if (!forward.body.includes(`${publicName}>${trustedName}`)) {
|
|
throw new Error(`Caddy proxy does not map authenticated ${label} identity`);
|
|
}
|
|
}
|
|
const outsideForward = route.body.slice(0, forward.start) + route.body.slice(forward.end + 1);
|
|
if (/X-Thoth-(?:Principal-[^\s>]+|Is-Admin)>X-Thoth-Trusted-/.test(outsideForward)) {
|
|
throw new Error("Caddy maps identity outside the authenticated response stage");
|
|
}
|
|
NODE
|
|
echo "Caddy reverse-proxy guide contract passed"
|
|
}
|
|
|
|
verify_caddy_adapted_identity_order() {
|
|
local adapted="$1"
|
|
node - "$adapted" <<'NODE'
|
|
const fs = require("fs");
|
|
const document = JSON.parse(fs.readFileSync(process.argv[2], "utf8"));
|
|
const publicHeaders = [
|
|
"X-Thoth-Principal-Issuer", "X-Thoth-Principal-Subject",
|
|
"X-Thoth-Principal-Display-Name", "X-Thoth-Is-Admin",
|
|
];
|
|
const trustedHeaders = [
|
|
"X-Thoth-Trusted-Principal-Issuer", "X-Thoth-Trusted-Principal-Subject",
|
|
"X-Thoth-Trusted-Principal-Display-Name", "X-Thoth-Trusted-Is-Admin",
|
|
];
|
|
function authUpstream(handler) {
|
|
return handler?.handler === "reverse_proxy" &&
|
|
(handler.upstreams || []).some((upstream) => upstream.dial === "auth-gateway:4180");
|
|
}
|
|
function frontendUpstream(handler) {
|
|
return handler?.handler === "reverse_proxy" &&
|
|
(handler.upstreams || []).some((upstream) => upstream.dial === "127.0.0.1:8080");
|
|
}
|
|
function collectTrustedSets(value, collected = new Map()) {
|
|
if (!value || typeof value !== "object") return collected;
|
|
if (value.handler === "headers") {
|
|
for (const [name, replacement] of Object.entries(value.request?.set || {})) {
|
|
if (trustedHeaders.includes(name)) collected.set(name, replacement);
|
|
}
|
|
}
|
|
for (const child of Object.values(value)) collectTrustedSets(child, collected);
|
|
return collected;
|
|
}
|
|
const expectedClears = [...publicHeaders, ...trustedHeaders];
|
|
function validateAuthenticatedMappings(auth) {
|
|
const successResponse = (auth.handle_response || []).find((response) =>
|
|
(response.match?.status_code || []).map(Number).includes(2));
|
|
if (!successResponse) throw new Error("Caddy adapted identity mapping is not restricted to auth 2xx");
|
|
const mappings = collectTrustedSets(successResponse);
|
|
for (let index = 0; index < trustedHeaders.length; index++) {
|
|
const replacement = mappings.get(trustedHeaders[index]);
|
|
const expected = `{http.reverse_proxy.header.${publicHeaders[index]}}`;
|
|
if (!Array.isArray(replacement) || replacement.length !== 1 || replacement[0] !== expected) {
|
|
throw new Error(`Caddy adapted authenticated mapping is invalid for ${trustedHeaders[index]}`);
|
|
}
|
|
}
|
|
}
|
|
function validateFrontendPath(handlers) {
|
|
let authAt = -1;
|
|
for (let index = handlers.length - 1; index >= 0; index--) {
|
|
if (authUpstream(handlers[index])) {
|
|
authAt = index;
|
|
break;
|
|
}
|
|
}
|
|
if (authAt < 0) {
|
|
throw new Error("Caddy adapted frontend path bypasses complete authentication contract");
|
|
}
|
|
for (const header of expectedClears) {
|
|
const clearAt = handlers.findIndex((handler) =>
|
|
handler?.handler === "headers" && (handler.request?.delete || []).includes(header));
|
|
if (clearAt < 0 || clearAt >= authAt) {
|
|
throw new Error("Caddy adapted identity clears must execute before authentication");
|
|
}
|
|
}
|
|
validateAuthenticatedMappings(handlers[authAt]);
|
|
for (let index = 0; index < handlers.length; index++) {
|
|
if (index !== authAt && collectTrustedSets(handlers[index]).size !== 0) {
|
|
throw new Error("Caddy adapted config maps trusted identity outside auth success");
|
|
}
|
|
}
|
|
}
|
|
let frontendPaths = 0;
|
|
function walk(value, inherited = []) {
|
|
if (!value || typeof value !== "object") return;
|
|
if (Array.isArray(value)) {
|
|
for (const child of value) walk(child, inherited);
|
|
return;
|
|
}
|
|
if (frontendUpstream(value)) {
|
|
frontendPaths++;
|
|
validateFrontendPath(inherited);
|
|
}
|
|
if (Array.isArray(value.handle)) {
|
|
const previous = [];
|
|
for (const handler of value.handle) {
|
|
walk(handler, [...inherited, ...previous]);
|
|
previous.push(handler);
|
|
}
|
|
for (const [key, child] of Object.entries(value)) {
|
|
if (key !== "handle") walk(child, inherited);
|
|
}
|
|
return;
|
|
}
|
|
const childContext = authUpstream(value) ? [...inherited, value] : inherited;
|
|
for (const child of Object.values(value)) walk(child, childContext);
|
|
}
|
|
walk(document);
|
|
if (frontendPaths === 0) {
|
|
throw new Error("Caddy adapted config lacks a frontend handler path");
|
|
}
|
|
NODE
|
|
}
|
|
|
|
verify_caddy_effective_proxy_guide() {
|
|
local adapted
|
|
adapted="$(mktemp "${TMPDIR:-/tmp}/thoth-caddy-adapted.XXXXXX")"
|
|
if ! awk '
|
|
/^```caddyfile$/ { code=1; next }
|
|
code && /^```$/ { exit }
|
|
code { print }
|
|
' "$root/docs/install/reverse-proxy-caddy.md" \
|
|
| docker run --rm -i caddy:2.10.2-alpine caddy adapt --config - --adapter caddyfile >"$adapted"; then
|
|
rm -f "$adapted"
|
|
echo "Caddy documented configuration could not be adapted" >&2
|
|
return 1
|
|
fi
|
|
verify_caddy_adapted_identity_order "$adapted"
|
|
rm -f "$adapted"
|
|
echo "Caddy adapted trust-stage contract passed"
|
|
}
|
|
|
|
verify_manual() {
|
|
local profile="$1" manual
|
|
manual="$root/docs/install/$profile-workspace-registry.md"
|
|
local -a headings
|
|
if [[ "$profile" == local ]]; then
|
|
headings=(
|
|
"Prerequisites"
|
|
"Git remote: SSH and HTTPS"
|
|
"Shared Git values, local bindings, and secret files"
|
|
"Direct PostgreSQL, REST, and SSH tunnel bindings"
|
|
"Bootstrap, first pull, and diagnostics"
|
|
"Publish, update, backup, outage recovery, and rollback"
|
|
"Troubleshooting"
|
|
)
|
|
else
|
|
headings=(
|
|
"Service account, storage, and firewall"
|
|
"Gitea and remote Git setup"
|
|
"Git credentials, CA, SSH key, and known-hosts mounts"
|
|
"Shared Git values, local bindings, and secret files"
|
|
"Direct PostgreSQL, REST, and SSH tunnel bindings"
|
|
"Same-origin reverse proxy, bootstrap, and health"
|
|
"Pull, publish, upgrade, backup, and recovery"
|
|
"Troubleshooting and snapshot rollback"
|
|
)
|
|
fi
|
|
for heading in "${headings[@]}"; do
|
|
grep -Fqx "## $heading" "$manual" || {
|
|
echo "missing required heading in $profile manual: $heading" >&2
|
|
return 1
|
|
}
|
|
done
|
|
local -a expected_steps
|
|
if [[ "$profile" == local ]]; then
|
|
expected_steps=(
|
|
'export THT_SOURCE_ROOT=/absolute/path/to/ThothII'
|
|
'--env-file "$THT_OPERATOR_ENV"'
|
|
"-f \"\$THT_SOURCE_ROOT/compose.yaml\" -f \"\$THT_SOURCE_ROOT/deploy/compose.$profile.yaml\""
|
|
'"$THT_SOURCE_ROOT/scripts/generate-connector-secrets-override.sh"'
|
|
)
|
|
else
|
|
expected_steps=(
|
|
'THTCTL=/srv/thothii/operator/thothctl'
|
|
'INSTALLATION=/srv/thothii/operator/thothii-installation.yaml'
|
|
'"$THTCTL" --installation "$INSTALLATION" start'
|
|
'"$THTCTL" --installation "$INSTALLATION" doctor'
|
|
'docs/install/examples/thothii-installation.server.yaml'
|
|
'compose.yaml'
|
|
'deploy/compose.server.yaml'
|
|
'server.md'
|
|
)
|
|
fi
|
|
for expected in "${expected_steps[@]}"; do
|
|
grep -Fq -- "$expected" "$manual" || {
|
|
echo "$profile manual lacks canonical operator step: $expected" >&2
|
|
return 1
|
|
}
|
|
done
|
|
if rg -n 'local-compose\.workspace-registry|server-compose\.workspace-registry|connector-secrets\.workspace-registry|docker compose' "$manual"; then
|
|
echo "$profile manual documents a superseded or bypassed Compose path" >&2
|
|
return 1
|
|
fi
|
|
verify_path_variable_values "$manual"
|
|
echo "$profile manual canonical base+override references passed"
|
|
}
|
|
|
|
verify_local_installation_example() {
|
|
local example="$root/docs/install/examples/thothii-installation.local.yaml"
|
|
[[ -f "$example" ]] || {
|
|
echo "missing local installation example: docs/install/examples/thothii-installation.local.yaml" >&2
|
|
return 1
|
|
}
|
|
|
|
local fixture source_copy operator_dir copied_example connector_override env_file
|
|
fixture="$(mktemp -d "${TMPDIR%/}/thoth local install.XXXXXX")"
|
|
trap 'rm -rf "$fixture"' RETURN
|
|
[[ "$fixture" == *" "* ]] || {
|
|
echo "local installation fixture path does not contain spaces" >&2
|
|
return 1
|
|
}
|
|
source_copy="$fixture/ThothII source"
|
|
operator_dir="$fixture/operator files"
|
|
mkdir -p "$source_copy/deploy/pi" "$operator_dir"
|
|
cp "$root/compose.yaml" "$source_copy/compose.yaml"
|
|
cp "$root/deploy/compose.local.yaml" "$source_copy/deploy/compose.local.yaml"
|
|
cp "$root/deploy/compose.git-ssh.yaml" "$source_copy/deploy/compose.git-ssh.yaml"
|
|
cp "$root/deploy/pi/models.json" "$source_copy/deploy/pi/models.json"
|
|
cp "$root/deploy/pi/settings.json" "$source_copy/deploy/pi/settings.json"
|
|
|
|
write_private "$operator_dir/pi-auth.json" '{"zai":{"type":"api_key","key":"fixture-local-pi-key"}}'
|
|
write_private "$operator_dir/thothii.secrets" 'THT_MODEL_API_KEY=fixture-local-model-key'
|
|
write_private "$operator_dir/git-ssh-key" 'fixture-local-ssh-key'
|
|
write_private "$operator_dir/git-known-hosts" 'fixture-local-known-hosts'
|
|
write_private "$operator_dir/dwh-password" 'fixture-local-dwh-password'
|
|
printf '%s\n' \
|
|
'THT_WS_NORTH_STAR_RESEARCH_DWH_TRANSPORT=postgres_direct' \
|
|
'THT_WS_NORTH_STAR_RESEARCH_DWH_PASSWORD_FILE=/run/secrets/north-star-research-dwh-password' \
|
|
>"$operator_dir/workspace-bindings.env"
|
|
env_file="$source_copy/deploy/env/local.env"
|
|
mkdir -p "$source_copy/deploy/env"
|
|
printf '%s\n' \
|
|
'THT_WORKSPACE_GIT_REMOTE=ssh://git@git.example.invalid/platform/thoth-workspaces.git' \
|
|
"PI_AUTH_FILE=$operator_dir/pi-auth.json" \
|
|
"THT_SECRETS_FILE=$operator_dir/thothii.secrets" \
|
|
"THT_WORKSPACE_BINDINGS_ENV_FILE=$operator_dir/workspace-bindings.env" \
|
|
"THT_WORKSPACE_GIT_SSH_KEY_FILE=$operator_dir/git-ssh-key" \
|
|
"THT_WORKSPACE_GIT_KNOWN_HOSTS_FILE=$operator_dir/git-known-hosts" \
|
|
"THT_WS_NORTH_STAR_RESEARCH_DWH_PASSWORD_SOURCE=$operator_dir/dwh-password" \
|
|
>"$env_file"
|
|
connector_override="$operator_dir/connector-secrets.local.yaml"
|
|
"$root/scripts/generate-connector-secrets-override.sh" \
|
|
--bindings-env "$operator_dir/workspace-bindings.env" \
|
|
--operator-env "$env_file" \
|
|
--output "$connector_override" >/dev/null
|
|
|
|
copied_example="$fixture/thothii-installation.yaml"
|
|
local contents
|
|
contents="$(<"$example")"
|
|
contents="${contents//\/absolute\/path\/to\/ThothII/$source_copy}"
|
|
contents="${contents//\/absolute\/path\/to\/thothii-operator/$operator_dir}"
|
|
printf '%s\n' "$contents" >"$copied_example"
|
|
|
|
local profile project_directory descriptor_env value
|
|
local -a overrides files
|
|
profile="$(sed -n 's/^profile: \([^[:space:]]*\)$/\1/p' "$copied_example")"
|
|
project_directory="$(sed -n 's/^projectDirectory: "\(.*\)"$/\1/p' "$copied_example")"
|
|
descriptor_env="$(sed -n 's/^envFile: "\(.*\)"$/\1/p' "$copied_example")"
|
|
while IFS= read -r value; do overrides+=("$value"); done < <(sed -n 's/^ - "\(.*\)"$/\1/p' "$copied_example")
|
|
[[ "$profile" == local && "$project_directory" == "$source_copy" && "$descriptor_env" == "$env_file" ]] || {
|
|
echo "local installation example does not resolve its required fields" >&2
|
|
return 1
|
|
}
|
|
[[ "${#overrides[@]}" -eq 2 && "${overrides[1]}" == "$connector_override" ]] || {
|
|
echo "local installation example does not select the expected optional overrides" >&2
|
|
return 1
|
|
}
|
|
files=(-f "$project_directory/compose.yaml" -f "$project_directory/deploy/compose.$profile.yaml")
|
|
for value in "${overrides[@]}"; do files+=(-f "$value"); done
|
|
local rendered="$fixture/local-installation.json"
|
|
"$root/scripts/compose-with-preflight.sh" --env-file "$descriptor_env" \
|
|
"${files[@]}" config --format json >"$rendered"
|
|
node - "$rendered" <<'NODE'
|
|
const fs = require("fs");
|
|
const config = JSON.parse(fs.readFileSync(process.argv[2], "utf8"));
|
|
if (Object.keys(config.services).sort().join(",") !== "core,embedding,embedding-model-init,frontend,qdrant") {
|
|
throw new Error("local installation example must render the internal semantic stack");
|
|
}
|
|
const output = JSON.stringify(config);
|
|
for (const secret of [
|
|
"fixture-local-pi-key",
|
|
"fixture-local-model-key",
|
|
"fixture-local-ssh-key",
|
|
"fixture-local-known-hosts",
|
|
"fixture-local-dwh-password",
|
|
]) {
|
|
if (output.includes(secret)) throw new Error("local installation rendering exposed a fixture secret");
|
|
}
|
|
NODE
|
|
echo "local installation example rendered from path with spaces passed"
|
|
}
|
|
|
|
verify_server_installation_example() {
|
|
local example="$root/docs/install/examples/thothii-installation.server.yaml"
|
|
[[ -f "$example" ]] || {
|
|
echo "missing server installation example: docs/install/examples/thothii-installation.server.yaml" >&2
|
|
return 1
|
|
}
|
|
|
|
local fixture source_copy operator_dir copied_example connector_override env_file backup_root
|
|
fixture="$(mktemp -d "${TMPDIR%/}/thoth server install.XXXXXX")"
|
|
trap 'rm -rf "$fixture"' RETURN
|
|
[[ "$fixture" == *" "* ]] || {
|
|
echo "server installation fixture path does not contain spaces" >&2
|
|
return 1
|
|
}
|
|
source_copy="$fixture/ThothII server source"
|
|
operator_dir="$fixture/server operator files"
|
|
backup_root="$fixture/server backups"
|
|
mkdir -p "$source_copy/deploy/pi" "$source_copy/deploy/workspaces" \
|
|
"$operator_dir/data" "$operator_dir/pi-state" "$operator_dir/workspace-registry" "$backup_root"
|
|
"$root/scripts/prepare-server-pi-state.sh" \
|
|
"$operator_dir/pi-state" "$(id -u)" "$(id -g)" >/dev/null
|
|
cp "$root/compose.yaml" "$source_copy/compose.yaml"
|
|
cp "$root/deploy/compose.server.yaml" "$source_copy/deploy/compose.server.yaml"
|
|
cp "$root/deploy/compose.session-server.yaml.example" \
|
|
"$source_copy/deploy/compose.session-server.yaml.example"
|
|
cp "$root/deploy/compose.git-ssh.yaml" "$source_copy/deploy/compose.git-ssh.yaml"
|
|
cp "$root/deploy/pi/models.json" "$source_copy/deploy/pi/models.json"
|
|
cp "$root/deploy/pi/settings.json" "$source_copy/deploy/pi/settings.json"
|
|
cp "$root/deploy/workspaces/server-sessions.yaml.example" \
|
|
"$source_copy/deploy/workspaces/server-sessions.yaml.example"
|
|
|
|
write_private "$operator_dir/pi-auth.json" '{"zai":{"type":"api_key","key":"fixture-server-pi-key"}}'
|
|
write_private "$operator_dir/thothii.secrets" 'THT_MODEL_API_KEY=fixture-server-model-key'
|
|
write_private "$operator_dir/git-ssh-key" 'fixture-server-ssh-key'
|
|
write_private "$operator_dir/git-known-hosts" 'fixture-server-known-hosts'
|
|
write_private "$operator_dir/dwh-password" 'fixture-server-dwh-password'
|
|
write_private "$operator_dir/session-runtime-password" 'fixture-server-session-runtime-password'
|
|
write_private "$operator_dir/session-migrator-password" 'fixture-server-session-migrator-password'
|
|
write_private "$operator_dir/session-ca.pem" 'fixture-server-session-ca'
|
|
printf '%s\n' \
|
|
'THT_WS_NORTH_STAR_RESEARCH_DWH_TRANSPORT=postgres_direct' \
|
|
'THT_WS_NORTH_STAR_RESEARCH_DWH_PASSWORD_FILE=/run/secrets/north-star-research-dwh-password' \
|
|
>"$operator_dir/workspace-bindings.env"
|
|
env_file="$operator_dir/server.env"
|
|
printf '%s\n' \
|
|
'THOTH_SERVER_BIND=127.0.0.1' \
|
|
'THOTH_HTTP_PORT=8080' \
|
|
'THT_WORKSPACE_GIT_REMOTE=ssh://git@git.example.invalid/platform/thoth-workspaces.git' \
|
|
'THT_WORKSPACE_GIT_BRANCH=main' \
|
|
"PI_AUTH_FILE=$operator_dir/pi-auth.json" \
|
|
"THT_SECRETS_FILE=$operator_dir/thothii.secrets" \
|
|
"THT_WORKSPACE_BINDINGS_ENV_FILE=$operator_dir/workspace-bindings.env" \
|
|
"THT_WORKSPACE_GIT_SSH_KEY_FILE=$operator_dir/git-ssh-key" \
|
|
"THT_WORKSPACE_GIT_KNOWN_HOSTS_FILE=$operator_dir/git-known-hosts" \
|
|
"THT_WS_NORTH_STAR_RESEARCH_DWH_PASSWORD_SOURCE=$operator_dir/dwh-password" \
|
|
"THT_DATA_ROOT=$operator_dir/data" \
|
|
"THT_PI_STATE_ROOT=$operator_dir/pi-state" \
|
|
"THT_WORKSPACE_REGISTRY_ROOT=$operator_dir/workspace-registry" \
|
|
"THT_BACKUP_ROOT=$backup_root" \
|
|
"THT_SERVER_WORKSPACE_CONFIG=$source_copy/deploy/workspaces/server-sessions.yaml.example" \
|
|
'THT_LLM_URL=https://llm.example.invalid' \
|
|
'THT_SESSION_DB_HOST=sessions.example.invalid' \
|
|
'THT_SESSION_DB_NAME=thoth_sessions' \
|
|
'THT_SESSION_RUNTIME_USER=thoth_sessions_app' \
|
|
'THT_SESSION_MIGRATOR_USER=thoth_sessions_migrate' \
|
|
"THT_SESSION_RUNTIME_PASSWORD_SOURCE=$operator_dir/session-runtime-password" \
|
|
"THT_SESSION_MIGRATOR_PASSWORD_SOURCE=$operator_dir/session-migrator-password" \
|
|
"THT_SESSION_CA_SOURCE=$operator_dir/session-ca.pem" \
|
|
>"$env_file"
|
|
connector_override="$operator_dir/connector-secrets.server.yaml"
|
|
"$root/scripts/generate-connector-secrets-override.sh" \
|
|
--bindings-env "$operator_dir/workspace-bindings.env" \
|
|
--operator-env "$env_file" \
|
|
--output "$connector_override" >/dev/null
|
|
|
|
copied_example="$fixture/thothii-installation.yaml"
|
|
local contents
|
|
contents="$(<"$example")"
|
|
contents="${contents//\/absolute\/path\/to\/ThothII/$source_copy}"
|
|
contents="${contents//\/absolute\/path\/to\/thothii-server-operator/$operator_dir}"
|
|
printf '%s\n' "$contents" >"$copied_example"
|
|
|
|
local profile project_directory descriptor_env value
|
|
local -a overrides files
|
|
profile="$(sed -n 's/^profile: \([^[:space:]]*\)$/\1/p' "$copied_example")"
|
|
project_directory="$(sed -n 's/^projectDirectory: "\(.*\)"$/\1/p' "$copied_example")"
|
|
descriptor_env="$(sed -n 's/^envFile: "\(.*\)"$/\1/p' "$copied_example")"
|
|
while IFS= read -r value; do overrides+=("$value"); done < <(sed -n 's/^ - "\(.*\)"$/\1/p' "$copied_example")
|
|
[[ "$profile" == server && "$project_directory" == "$source_copy" && "$descriptor_env" == "$env_file" ]] || {
|
|
echo "server installation example does not resolve its required fields" >&2
|
|
return 1
|
|
}
|
|
[[ "${#overrides[@]}" -eq 3 && "${overrides[0]}" == "$source_copy/deploy/compose.session-server.yaml.example" \
|
|
&& "${overrides[2]}" == "$connector_override" ]] || {
|
|
echo "server installation example does not select the expected optional overrides" >&2
|
|
return 1
|
|
}
|
|
files=(-f "$project_directory/compose.yaml" -f "$project_directory/deploy/compose.$profile.yaml")
|
|
for value in "${overrides[@]}"; do files+=(-f "$value"); done
|
|
local rendered="$fixture/server-installation.json"
|
|
"$root/scripts/compose-with-preflight.sh" --env-file "$descriptor_env" \
|
|
"${files[@]}" config --format json >"$rendered"
|
|
node - "$rendered" <<'NODE'
|
|
const fs = require("fs");
|
|
const config = JSON.parse(fs.readFileSync(process.argv[2], "utf8"));
|
|
if (Object.keys(config.services).sort().join(",") !== "core,embedding,embedding-model-init,frontend,qdrant") {
|
|
throw new Error("server installation example must render the internal semantic stack");
|
|
}
|
|
const core = config.services.core;
|
|
const frontend = config.services.frontend;
|
|
if (core.environment?.AUTH_MODE !== "upstream" || core.environment?.THOTH_PUBLIC_EXPOSURE !== "true") {
|
|
throw new Error("server installation example must fail closed behind upstream authentication");
|
|
}
|
|
if ((core.ports || []).length !== 0) throw new Error("server installation example published core");
|
|
const ports = frontend.ports || [];
|
|
if (ports.length !== 1 || ports[0].host_ip !== "127.0.0.1" || Number(ports[0].target) !== 8080) {
|
|
throw new Error("server installation example must publish only loopback frontend");
|
|
}
|
|
const rendered = JSON.stringify(config);
|
|
if (/omics_portal|chirone|localllm_default|datamart-builder/i.test(rendered)) {
|
|
throw new Error("server installation example contains application coupling");
|
|
}
|
|
for (const secret of [
|
|
"fixture-server-pi-key", "fixture-server-model-key", "fixture-server-ssh-key",
|
|
"fixture-server-known-hosts", "fixture-server-dwh-password",
|
|
"fixture-server-session-runtime-password", "fixture-server-session-migrator-password",
|
|
"fixture-server-session-ca",
|
|
]) {
|
|
if (rendered.includes(secret)) throw new Error("server installation rendering exposed a fixture secret");
|
|
}
|
|
NODE
|
|
echo "server installation example rendered from path with spaces passed"
|
|
|
|
local migration_rendered="$fixture/server-migration.json"
|
|
"$root/scripts/compose-with-preflight.sh" --env-file "$descriptor_env" \
|
|
"${files[@]}" --profile session-migrate config --format json >"$migration_rendered"
|
|
node - "$migration_rendered" <<'NODE'
|
|
const fs = require("fs");
|
|
const config = JSON.parse(fs.readFileSync(process.argv[2], "utf8"));
|
|
const services = config.services || {};
|
|
if (!services.core || !services["session-migrate"]) throw new Error("server migration profile is missing core or session-migrate");
|
|
if (services.core.image !== services["session-migrate"].image) throw new Error("source migration image differs from core");
|
|
if (services["session-migrate"].build) throw new Error("source migration service unexpectedly declares a build");
|
|
NODE
|
|
|
|
local pinned_template="$fixture/pinned-template.yaml" pinned_override="$operator_dir/pinned-images.yaml"
|
|
awk '
|
|
/^## Build locally or select pinned images$/ { section=1; next }
|
|
section && /^```yaml$/ { code=1; next }
|
|
code && /^```$/ { exit }
|
|
code { print }
|
|
' "$root/docs/install/server.md" >"$pinned_template"
|
|
sed \
|
|
-e "s#registry.example.com/thothii/core@sha256:<64-lowercase-hex-digits>#registry.example.com/thothii/core@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa#g" \
|
|
-e "s#registry.example.com/thothii/frontend@sha256:<64-lowercase-hex-digits>#registry.example.com/thothii/frontend@sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb#g" \
|
|
"$pinned_template" >"$pinned_override"
|
|
chmod 0600 "$pinned_override"
|
|
local pinned_rendered="$fixture/server-pinned-migration.json"
|
|
"$root/scripts/compose-with-preflight.sh" --env-file "$descriptor_env" \
|
|
"${files[@]}" -f "$pinned_override" --profile session-migrate config --format json >"$pinned_rendered"
|
|
node - "$pinned_rendered" <<'NODE'
|
|
const fs = require("fs");
|
|
const config = JSON.parse(fs.readFileSync(process.argv[2], "utf8"));
|
|
const core = config.services?.core;
|
|
const frontend = config.services?.frontend;
|
|
const migrator = config.services?.["session-migrate"];
|
|
if (!core || !frontend || !migrator) throw new Error("pinned migration profile lacks core, frontend, or session-migrate");
|
|
if (core.image !== migrator.image || !/@sha256:[a-f0-9]{64}$/.test(core.image)) {
|
|
throw new Error("pinned migration image does not equal the exact core digest");
|
|
}
|
|
if (!/@sha256:[a-f0-9]{64}$/.test(frontend.image)) throw new Error("frontend is not pinned by exact digest");
|
|
for (const [name, service] of Object.entries({core, frontend, migrator})) {
|
|
if (service.build) throw new Error(name + " retained a local build in pinned mode");
|
|
if (/:local$/.test(service.image || "")) throw new Error(name + " retained a local image in pinned mode");
|
|
}
|
|
NODE
|
|
echo "server pinned migration image fixture passed"
|
|
|
|
local checksum_root="$fixture/root-only-checksum"
|
|
mkdir -m 0700 "$checksum_root"
|
|
printf 'fixture backup bytes\n' >"$checksum_root/runtime-data.tgz"
|
|
/bin/sh -ceu 'cd "$1"; sha256sum runtime-data.tgz > SHA256SUMS; sha256sum --check SHA256SUMS' sh "$checksum_root" >/dev/null
|
|
printf 'corruption\n' >>"$checksum_root/runtime-data.tgz"
|
|
if (cd "$checksum_root" && sha256sum --check SHA256SUMS) >/dev/null 2>&1; then
|
|
echo "corrupted server backup checksum fixture was accepted" >&2
|
|
return 1
|
|
fi
|
|
echo "server backup checksum root-only fixture passed"
|
|
}
|
|
|
|
write_private() {
|
|
local path="$1" value="$2"
|
|
printf '%s\n' "$value" >"$path"
|
|
chmod 0600 "$path"
|
|
}
|
|
|
|
verify_compose_fixtures() {
|
|
local fixture connector_override profile rendered
|
|
fixture="$(mktemp -d "${TMPDIR%/}/thoth-install-fixtures.XXXXXX")"
|
|
trap 'rm -rf "$fixture"' RETURN
|
|
mkdir -p "$fixture/data" "$fixture/pi-state" "$fixture/workspace-registry"
|
|
|
|
write_private "$fixture/pi-auth.json" '{"zai":{"type":"api_key","key":"fixture-native-auth-key"}}'
|
|
write_private "$fixture/thothii.secrets" 'THT_MODEL_API_KEY=fixture-model-api-key'
|
|
write_private "$fixture/git-ssh-key" 'fixture-git-ssh-key'
|
|
write_private "$fixture/git-known-hosts" 'fixture-git-known-hosts'
|
|
write_private "$fixture/dwh-password" 'fixture-dwh-password'
|
|
write_private "$fixture/session-runtime-password" 'fixture-session-runtime-password'
|
|
write_private "$fixture/session-migrator-password" 'fixture-session-migrator-password'
|
|
write_private "$fixture/session-ca.pem" 'fixture-session-ca'
|
|
cp "$root/deploy/workspaces/server-sessions.yaml.example" "$fixture/server-sessions.yaml"
|
|
|
|
printf '%s\n' \
|
|
'THT_WS_NORTH_STAR_RESEARCH_DWH_TRANSPORT=postgres_direct' \
|
|
'THT_WS_NORTH_STAR_RESEARCH_DWH_PASSWORD_FILE=/run/secrets/north-star-research-dwh-password' \
|
|
>"$fixture/workspace-bindings.env"
|
|
|
|
printf '%s\n' \
|
|
'THT_WORKSPACE_GIT_REMOTE=ssh://git@git.example.invalid/platform/thoth-workspaces.git' \
|
|
"PI_AUTH_FILE=$fixture/pi-auth.json" \
|
|
"THT_SECRETS_FILE=$fixture/thothii.secrets" \
|
|
"THT_WORKSPACE_BINDINGS_ENV_FILE=$fixture/workspace-bindings.env" \
|
|
"THT_WORKSPACE_GIT_SSH_KEY_FILE=$fixture/git-ssh-key" \
|
|
"THT_WORKSPACE_GIT_KNOWN_HOSTS_FILE=$fixture/git-known-hosts" \
|
|
"THT_WS_NORTH_STAR_RESEARCH_DWH_PASSWORD_SOURCE=$fixture/dwh-password" \
|
|
"THT_DATA_ROOT=$fixture/data" \
|
|
"THT_PI_STATE_ROOT=$fixture/pi-state" \
|
|
"THT_WORKSPACE_REGISTRY_ROOT=$fixture/workspace-registry" \
|
|
"THT_SERVER_WORKSPACE_CONFIG=$fixture/server-sessions.yaml" \
|
|
'THT_SESSION_DB_HOST=sessions.example.invalid' \
|
|
'THT_SESSION_DB_NAME=thoth_sessions' \
|
|
'THT_SESSION_RUNTIME_USER=thoth_sessions_app' \
|
|
'THT_SESSION_MIGRATOR_USER=thoth_sessions_migrate' \
|
|
"THT_SESSION_RUNTIME_PASSWORD_SOURCE=$fixture/session-runtime-password" \
|
|
"THT_SESSION_MIGRATOR_PASSWORD_SOURCE=$fixture/session-migrator-password" \
|
|
"THT_SESSION_CA_SOURCE=$fixture/session-ca.pem" \
|
|
>"$fixture/operator.env"
|
|
|
|
connector_override="$fixture/connector-secrets.local.yaml"
|
|
"$root/scripts/generate-connector-secrets-override.sh" \
|
|
--bindings-env "$fixture/workspace-bindings.env" \
|
|
--operator-env "$fixture/operator.env" \
|
|
--output "$connector_override" >/dev/null
|
|
|
|
for profile in local server; do
|
|
rendered="$fixture/$profile.json"
|
|
files=(
|
|
-f "$root/compose.yaml"
|
|
-f "$root/deploy/compose.$profile.yaml"
|
|
)
|
|
if [[ "$profile" == server ]]; then
|
|
files+=(-f "$root/deploy/compose.session-server.yaml.example")
|
|
fi
|
|
files+=(
|
|
-f "$root/deploy/compose.git-ssh.yaml"
|
|
-f "$connector_override"
|
|
)
|
|
"$root/scripts/compose-with-preflight.sh" --env-file "$fixture/operator.env" \
|
|
"${files[@]}" config --format json >"$rendered"
|
|
|
|
node - "$rendered" "$profile" <<'NODE'
|
|
const fs = require("fs");
|
|
const [path, profile] = process.argv.slice(2);
|
|
const config = JSON.parse(fs.readFileSync(path, "utf8"));
|
|
if (Object.keys(config.services).sort().join(",") !== "core,embedding,embedding-model-init,frontend,qdrant") {
|
|
throw new Error(profile + ": mandatory stack must include the internal semantic services");
|
|
}
|
|
const core = config.services.core;
|
|
for (const target of [
|
|
"/home/thoth/.pi/agent/auth.json",
|
|
"/home/thoth/.pi/agent/models.json",
|
|
"/home/thoth/.pi/agent/settings.json",
|
|
]) {
|
|
if (!(core.volumes || []).some((mount) => mount.target === target && mount.read_only)) {
|
|
throw new Error(profile + ": missing read-only Pi mount " + target);
|
|
}
|
|
}
|
|
for (const [name, value] of Object.entries({
|
|
THT_WS_NORTH_STAR_RESEARCH_DWH_PASSWORD_FILE: "/run/secrets/north-star-research-dwh-password",
|
|
})) {
|
|
if (core.environment?.[name] !== value) throw new Error(profile + ": missing binding " + name);
|
|
}
|
|
const secretTargets = new Set((core.secrets || []).map((secret) => secret.target));
|
|
for (const target of [
|
|
"thothii.secrets",
|
|
"north-star-research-dwh-password",
|
|
]) {
|
|
if (!secretTargets.has(target)) throw new Error(profile + ": missing secret target " + target);
|
|
}
|
|
if (profile === "server") {
|
|
for (const target of ["session_runtime_password", "session_ca.pem"]) {
|
|
if (!secretTargets.has(target)) throw new Error("server: missing session secret " + target);
|
|
}
|
|
}
|
|
if ((config.services.frontend.secrets || []).length !== 0) {
|
|
throw new Error(profile + ": frontend received a runtime secret");
|
|
}
|
|
const rendered = JSON.stringify(config);
|
|
for (const value of [
|
|
"fixture-native-auth-key", "fixture-model-api-key", "fixture-git-ssh-key",
|
|
"fixture-git-known-hosts", "fixture-dwh-password",
|
|
"fixture-session-runtime-password", "fixture-session-migrator-password", "fixture-session-ca",
|
|
]) {
|
|
if (rendered.includes(value)) throw new Error(profile + ": rendered Compose leaked " + value);
|
|
}
|
|
NODE
|
|
echo "canonical $profile base+override fixture passed"
|
|
done
|
|
|
|
printf 'THT_WS_EXAMPLE_DWH_PASSWORD_SOURCE=relative/secret\n' >"$fixture/unsafe.env"
|
|
if verify_path_variable_values "$fixture/unsafe.env" >/dev/null 2>&1; then
|
|
echo "relative secret-source fixture was accepted" >&2
|
|
return 1
|
|
fi
|
|
echo "relative secret-source fixture rejected passed"
|
|
}
|
|
|
|
case "$mode" in
|
|
--fixtures-only)
|
|
[[ $# -eq 1 ]] || { echo "usage: $0 --fixtures-only" >&2; exit 2; }
|
|
verify_local_guide
|
|
verify_windows_line_endings_guide
|
|
verify_pi_management_guide
|
|
verify_server_guide
|
|
verify_reverse_proxy_nginx_guide
|
|
verify_reverse_proxy_caddy_guide
|
|
verify_local_installation_example
|
|
verify_server_installation_example
|
|
verify_manual local
|
|
verify_manual server
|
|
verify_compose_fixtures
|
|
;;
|
|
--profile)
|
|
profile="${2:-}"
|
|
[[ $# -eq 2 && "$profile" =~ ^(local|server)$ ]] \
|
|
|| { echo "usage: $0 --profile {local|server}" >&2; exit 2; }
|
|
if [[ "$profile" == local ]]; then
|
|
verify_local_guide
|
|
verify_windows_line_endings_guide
|
|
verify_pi_management_guide
|
|
verify_local_installation_example
|
|
else
|
|
verify_server_guide
|
|
verify_reverse_proxy_nginx_guide
|
|
verify_reverse_proxy_caddy_guide
|
|
verify_caddy_effective_proxy_guide
|
|
"$root/scripts/test-server-operator-permissions.sh"
|
|
verify_server_installation_example
|
|
fi
|
|
verify_manual "$profile"
|
|
verify_compose_fixtures
|
|
echo "== Run isolated workspace-registry bootstrap and recovery smoke =="
|
|
(
|
|
cd "$root"
|
|
env -u WORKSPACE_GIT_REMOTE ./scripts/workspace-registry-smoke.sh
|
|
)
|
|
echo "$profile installation documentation verification passed"
|
|
;;
|
|
*)
|
|
echo "usage: $0 --fixtures-only | --profile {local|server}" >&2
|
|
exit 2
|
|
;;
|
|
esac
|