mptyl/ThothII
1
0
Fork 0
Code Issues 25 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

fix: make local recovery fail closed

Browse Source
This commit is contained in:
marcopan
2026-08-05 09:48:55 +02:00
parent 65aa52115f
commit df21046472
4 changed files with 577 additions and 115 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/install/local.md
+125 -57
View File
@@ -245,90 +245,158 @@ selected by the durable, installation-specific `current-image.yaml` after every
Therefore rebuilding `thothii-core:local` followed by `update --check-only` does not reconcile a
previous `pi update`: the old promoted core would remain selected.
Do not delete or edit the selector. The supported source-update path is a transactional
`pi update --source build` from a clean pulled checkout whose `docker/core.Dockerfile` pins a
different Pi version than the running installation. `thothctl` currently treats the same requested
Pi version as a no-op. The explicit comparison below therefore stops instead of silently deploying
only part of a revision. If it stops, keep the current installation running and wait for a release
with a new Pi pin or a future supported reconciliation command; there is no supported manual
same-version selector-removal procedure.
Do not delete or edit the selector. `thothctl status` is the installation-aware selector test. If
the running core image is the base `thothii-core:local` image, no Pi update has promoted a durable
lifecycle image and an ordinary same-Pi-version source rebuild/start is supported. If status shows
a lifecycle image and the pulled Pi pin is unchanged, `pi update` would be a no-op and the procedure
must stop. A changed Pi pin uses transactional `pi update --source build` in either case.
macOS, Linux, and WSL2:
```sh
git status --short
git diff --quiet
git diff --cached --quiet
git pull --ff-only
git config --local core.autocrlf false
bash scripts/verify-line-endings.sh
SOURCE_REVISION="$(git rev-parse HEAD)"
NEXT_PI_VERSION="$(sed -n 's/^ARG PI_VERSION=//p' docker/core.Dockerfile)"
RUNNING_PI_VERSION="$("$THTCTL" --installation "$INSTALLATION" pi status)"
RUNNING_PI_VERSION="${RUNNING_PI_VERSION#Pi version: }"
if [[ -z "$NEXT_PI_VERSION" || "$NEXT_PI_VERSION" == "$RUNNING_PI_VERSION" ]]; then
echo "Source update stopped: the pulled revision must pin a new Pi version." >&2
exit 1
set -euo pipefail
abort_update() { printf 'Source update stopped: %s\n' "$1" >&2; exit 1; }
require_clean_source() {
local source_state
if ! source_state="$(git status --porcelain --untracked-files=all)"; then
abort_update "git status failed"
fi
[[ -z "$source_state" ]] || abort_update "commit, remove, or back up every tracked/untracked source change"
}
require_clean_source
if ! git pull --ff-only; then abort_update "git pull --ff-only failed"; fi
require_clean_source
if ! git config --local core.autocrlf false; then abort_update "could not set repository LF policy"; fi
if ! bash scripts/verify-line-endings.sh; then abort_update "the pulled checkout contains CRLF files"; fi
if ! SOURCE_REVISION="$(git rev-parse HEAD)"; then abort_update "could not record the pulled revision"; fi
if ! NEXT_PI_VERSION="$(sed -n 's/^ARG PI_VERSION=//p' docker/core.Dockerfile)"; then
abort_update "could not read the pulled Pi pin"
fi
bash scripts/build-local.sh
bash scripts/build-thothctl.sh
"$THTCTL" --installation "$INSTALLATION" update --check-only
"$THTCTL" --installation "$INSTALLATION" pi update \
--version "$NEXT_PI_VERSION" --source build --yes --drain
"$THTCTL" --installation "$INSTALLATION" start
curl --fail http://127.0.0.1:8080/health
curl --fail http://127.0.0.1:8787/health
printf 'Built source revision: %s\n' "$SOURCE_REVISION"
"$THTCTL" --installation "$INSTALLATION" status
"$THTCTL" --installation "$INSTALLATION" pi status
"$THTCTL" --installation "$INSTALLATION" doctor
[[ -n "$NEXT_PI_VERSION" && "$NEXT_PI_VERSION" != *$'\n'* ]] || abort_update "expected one pinned default PI_VERSION"
if ! INSTALLATION_STATUS="$("$THTCTL" --installation "$INSTALLATION" status)"; then
abort_update "thothctl status failed"
fi
if ! RUNNING_PI_VERSION="$("$THTCTL" --installation "$INSTALLATION" pi status)"; then
abort_update "thothctl pi status failed"
fi
RUNNING_PI_VERSION="${RUNNING_PI_VERSION#Pi version: }"
[[ -n "$RUNNING_PI_VERSION" ]] || abort_update "thothctl pi status returned no version"
COMPACT_STATUS="${INSTALLATION_STATUS//[[:space:]]/}"
USES_BASE_CORE=false
if [[ "$COMPACT_STATUS" == *'"Image":"thothii-core:local"'* ]]; then
USES_BASE_CORE=true
fi
TRANSACTIONAL_PI_UPDATE=true
if [[ "$NEXT_PI_VERSION" == "$RUNNING_PI_VERSION" ]]; then
[[ "$USES_BASE_CORE" == true ]] || abort_update "same Pi version is selected by a durable lifecycle image"
TRANSACTIONAL_PI_UPDATE=false
fi
if ! bash scripts/build-local.sh; then abort_update "the local image build failed"; fi
if ! bash scripts/build-thothctl.sh; then abort_update "the thothctl build failed"; fi
if ! "$THTCTL" --installation "$INSTALLATION" update --check-only; then
abort_update "the installation render check failed"
fi
if [[ "$TRANSACTIONAL_PI_UPDATE" == true ]]; then
if ! "$THTCTL" --installation "$INSTALLATION" pi update \
--version "$NEXT_PI_VERSION" --source build --yes --drain; then
abort_update "the transactional core update failed"
fi
fi
if ! "$THTCTL" --installation "$INSTALLATION" start; then abort_update "installation start failed"; fi
if ! curl --fail http://127.0.0.1:8080/health; then abort_update "frontend health check failed"; fi
if ! curl --fail http://127.0.0.1:8787/health; then abort_update "core health check failed"; fi
if ! FINAL_STATUS="$("$THTCTL" --installation "$INSTALLATION" status)"; then abort_update "final status failed"; fi
if ! FINAL_PI_STATUS="$("$THTCTL" --installation "$INSTALLATION" pi status)"; then abort_update "final pi status failed"; fi
[[ "${FINAL_PI_STATUS#Pi version: }" == "$NEXT_PI_VERSION" ]] || abort_update "running Pi version does not match the pulled pin"
if ! "$THTCTL" --installation "$INSTALLATION" doctor; then abort_update "final doctor failed"; fi
require_clean_source
printf 'Built source revision: %s\n%s\n%s\n' "$SOURCE_REVISION" "$FINAL_STATUS" "$FINAL_PI_STATUS"
```
Native Windows PowerShell uses the same fail-closed version comparison and transactional promotion:
```powershell
git status --short
git diff --quiet
if ($LASTEXITCODE -ne 0) { throw 'Commit or back up tracked source changes before update.' }
git diff --cached --quiet
if ($LASTEXITCODE -ne 0) { throw 'Commit or back up staged source changes before update.' }
$ErrorActionPreference = 'Stop'
function Assert-NativeSuccess([string]$Step) {
if ($LASTEXITCODE -ne 0) { throw "$Step failed with exit code $LASTEXITCODE." }
}
function Assert-CleanSource {
$SourceState = @(git status --porcelain --untracked-files=all)
Assert-NativeSuccess 'git status'
if ($SourceState.Count -ne 0) {
throw 'Commit, remove, or back up every tracked/untracked source change.'
}
}
Assert-CleanSource
git pull --ff-only
if ($LASTEXITCODE -ne 0) { throw 'The source pull failed.' }
Assert-NativeSuccess 'source pull'
Assert-CleanSource
git config --local core.autocrlf false
Assert-NativeSuccess 'repository LF policy'
& "C:\Program Files\Git\bin\bash.exe" scripts/verify-line-endings.sh
if ($LASTEXITCODE -ne 0) { throw 'The pulled checkout contains CRLF files.' }
Assert-NativeSuccess 'pulled checkout LF verification'
$SourceRevision = git rev-parse HEAD
Assert-NativeSuccess 'source revision read'
$VersionLine = @(Select-String -Path docker/core.Dockerfile -Pattern '^ARG PI_VERSION=(.+)$')
if ($VersionLine.Count -ne 1) { throw 'Expected exactly one pinned default PI_VERSION.' }
$NextPiVersion = $VersionLine.Matches[0].Groups[1].Value
$RunningPiVersion = (& $THTCTL --installation $INSTALLATION pi status) `
-replace '^Pi version:\s*', ''
if ([string]::IsNullOrWhiteSpace($NextPiVersion) -or $NextPiVersion -eq $RunningPiVersion) {
throw 'Source update stopped: the pulled revision must pin a new Pi version.'
$InstallationStatus = @(& $THTCTL --installation $INSTALLATION status)
Assert-NativeSuccess 'installation status'
$RunningPiStatus = (& $THTCTL --installation $INSTALLATION pi status)
Assert-NativeSuccess 'Pi status'
$RunningPiVersion = $RunningPiStatus -replace '^Pi version:\s*', ''
if ([string]::IsNullOrWhiteSpace($RunningPiVersion)) { throw 'Pi status returned no version.' }
$Services = $InstallationStatus | ConvertFrom-Json
$CoreServices = @($Services | Where-Object { $_.Service -eq 'core' })
if ($CoreServices.Count -ne 1) { throw 'Installation status did not identify exactly one core service.' }
$UsesBaseCore = $CoreServices[0].Image -eq 'thothii-core:local'
$TransactionalPiUpdate = $true
if ($NextPiVersion -eq $RunningPiVersion) {
if (-not $UsesBaseCore) { throw 'Same Pi version is selected by a durable lifecycle image.' }
$TransactionalPiUpdate = $false
}
powershell -ExecutionPolicy Bypass -File scripts/build-local.ps1
if ($LASTEXITCODE -ne 0) { throw 'The local image build failed.' }
Assert-NativeSuccess 'local image build'
& "C:\Program Files\Git\bin\bash.exe" scripts/build-thothctl.sh
if ($LASTEXITCODE -ne 0) { throw 'The thothctl build failed.' }
Assert-NativeSuccess 'thothctl build'
& $THTCTL --installation $INSTALLATION update --check-only
if ($LASTEXITCODE -ne 0) { throw 'The installation render check failed.' }
& $THTCTL --installation $INSTALLATION pi update `
--version $NextPiVersion --source build --yes --drain
if ($LASTEXITCODE -ne 0) { throw 'The transactional core update failed.' }
Assert-NativeSuccess 'installation render check'
if ($TransactionalPiUpdate) {
& $THTCTL --installation $INSTALLATION pi update `
--version $NextPiVersion --source build --yes --drain
Assert-NativeSuccess 'transactional core update'
}
& $THTCTL --installation $INSTALLATION start
if ($LASTEXITCODE -ne 0) { throw 'The installation start failed.' }
Assert-NativeSuccess 'installation start'
curl.exe --fail --silent --show-error http://127.0.0.1:8080/health
Assert-NativeSuccess 'frontend health check'
curl.exe --fail --silent --show-error http://127.0.0.1:8787/health
Write-Output "Built source revision: $SourceRevision"
& $THTCTL --installation $INSTALLATION status
& $THTCTL --installation $INSTALLATION pi status
Assert-NativeSuccess 'core health check'
$FinalStatus = @(& $THTCTL --installation $INSTALLATION status)
Assert-NativeSuccess 'final installation status'
$FinalPiStatus = (& $THTCTL --installation $INSTALLATION pi status)
Assert-NativeSuccess 'final Pi status'
if (($FinalPiStatus -replace '^Pi version:\s*', '') -ne $NextPiVersion) {
throw 'Running Pi version does not match the pulled pin.'
}
& $THTCTL --installation $INSTALLATION doctor
Assert-NativeSuccess 'final doctor'
Assert-CleanSource
Write-Output "Built source revision: $SourceRevision"
Write-Output $FinalStatus
Write-Output $FinalPiStatus
```
The recorded Git revision identifies the clean worktree used for the candidate build. In
`thothctl status`, confirm that `core` reports the installation lifecycle candidate image, then
require `pi status` to equal the new pin and `doctor` to pass. This is the supported running-image
and source-revision evidence; `update --check-only` alone proves only that Compose renders.
The revision is printed only after every source/build/start/health/installation-aware check passes
and a final porcelain check still reports no tracked or untracked source changes. For a changed Pi
pin, status reports the promoted lifecycle candidate; for a same-version installation with no
selector, status reports the rebuilt base core. `update --check-only` alone proves only that Compose
renders.
Review release notes before updating. See [Pi management](pi-management.md) for rollback; never
install a package in the running container.
docs/install/windows-line-endings.md
+145 -24
View File
@@ -77,53 +77,174 @@ worktree bytes.
From WSL2, Git Bash, macOS, or Linux:
```sh
git status --short
git config --local core.autocrlf false
git add --renormalize .
git diff --cached --check
git diff --cached
set -euo pipefail
abort_repair() { printf 'CRLF repair stopped: %s\n' "$1" >&2; exit 1; }
validate_index_export() {
git ls-files -s -z | while IFS= read -r -d '' entry; do
metadata="${entry%%$'\t'*}"
path="${entry#*$'\t'}"
mode="${metadata%% *}"
[[ "$path" != "$entry" ]] || exit 1
case "$mode" in
100644|100755) [[ -f "$REPAIR_DIR/$path" && ! -L "$REPAIR_DIR/$path" ]] || exit 1 ;;
120000) [[ -L "$REPAIR_DIR/$path" ]] && readlink "$REPAIR_DIR/$path" >/dev/null || exit 1 ;;
*) printf 'Unsupported Git mode %s: %s\n' "$mode" "$path" >&2; exit 1 ;;
esac
done
}
validate_worktree_modes() {
git ls-files -s -z | while IFS= read -r -d '' entry; do
metadata="${entry%%$'\t'*}"
path="${entry#*$'\t'}"
mode="${metadata%% *}"
case "$mode" in
100644|100755) [[ -f "$path" && ! -L "$path" ]] || exit 1 ;;
120000) [[ -L "$path" ]] && readlink "$path" >/dev/null || exit 1 ;;
*) exit 1 ;;
esac
done
}
rewrite_index_entry() {
local mode="$1" path="$2" target temporary_link
case "$mode" in
100644)
cp "$REPAIR_DIR/$path" "$path" && chmod a-x "$path"
;;
100755)
cp "$REPAIR_DIR/$path" "$path" && chmod a+x "$path"
;;
120000)
target="$(readlink "$REPAIR_DIR/$path")" || return 1
temporary_link="${path}.thoth-lf-repair-link"
[[ ! -e "$temporary_link" && ! -L "$temporary_link" ]] || return 1
ln -s "$target" "$temporary_link" || return 1
rm -f "$path" || { rm -f "$temporary_link"; return 1; }
mv "$temporary_link" "$path"
;;
*) return 1 ;;
esac
}
if ! git status --short; then abort_repair "git status failed"; fi
if ! git config --local core.autocrlf false; then abort_repair "could not set repository LF policy"; fi
if ! git add --renormalize .; then abort_repair "index renormalization failed"; fi
if ! git diff --cached --check; then abort_repair "normalized index check failed"; fi
if ! git diff --cached; then abort_repair "normalized index review failed"; fi
REPAIR_DIR="$(cd .. && pwd -P)/ThothII-lf-repair"
if [[ -e "$REPAIR_DIR" ]]; then
echo "Choose a new empty LF repair directory: $REPAIR_DIR" >&2
exit 1
abort_repair "choose a new empty LF repair directory: $REPAIR_DIR"
fi
mkdir -p "$REPAIR_DIR"
if ! mkdir -p "$REPAIR_DIR"; then abort_repair "could not create LF repair directory"; fi
REPAIR_PREFIX="$REPAIR_DIR/"
git checkout-index --all --force --prefix="$REPAIR_PREFIX"
bash scripts/verify-line-endings.sh "$REPAIR_DIR"
if ! git checkout-index --all --force --prefix="$REPAIR_PREFIX"; then abort_repair "index export failed"; fi
if ! validate_index_export; then abort_repair "index export is missing entries or Git modes"; fi
if ! bash scripts/verify-line-endings.sh "$REPAIR_DIR"; then abort_repair "exported bytes failed LF verification"; fi
# WARNING: destructive copy; make a backup or commit wanted changes before this command.
git ls-files -z | while IFS= read -r -d '' path; do
cp "$REPAIR_DIR/$path" "$path"
done
bash scripts/verify-line-endings.sh
if ! git ls-files -s -z | while IFS= read -r -d '' entry; do
metadata="${entry%%$'\t'*}"
path="${entry#*$'\t'}"
mode="${metadata%% *}"
rewrite_index_entry "$mode" "$path" || exit 1
done; then
abort_repair "tracked-file rewrite failed; do not build from this worktree"
fi
if ! validate_worktree_modes; then abort_repair "repaired worktree does not match Git index modes"; fi
if ! bash scripts/verify-line-endings.sh; then abort_repair "repaired worktree failed LF verification"; fi
if ! git diff --cached --check; then abort_repair "repaired index check failed"; fi
```
Native Windows PowerShell runs the same Git operations and invokes the byte verifier through Git
for Windows:
```powershell
$ErrorActionPreference = 'Stop'
function Assert-NativeSuccess([string]$Step) {
if ($LASTEXITCODE -ne 0) { throw "$Step failed with exit code $LASTEXITCODE." }
}
function ConvertFrom-IndexEntry([string]$Entry) {
if ($Entry -notmatch '^([0-9]{6}) [0-9a-f]+ [0-3]\t(.+)$') {
throw "Invalid Git index entry: $Entry"
}
[pscustomobject]@{ Mode = $Matches[1]; Path = $Matches[2] }
}
git status --short
Assert-NativeSuccess 'git status'
git config --local core.autocrlf false
Assert-NativeSuccess 'repository LF policy'
git add --renormalize .
Assert-NativeSuccess 'index renormalization'
git diff --cached --check
Assert-NativeSuccess 'normalized index check'
git diff --cached
Assert-NativeSuccess 'normalized index review'
$RepairDir = Join-Path (Split-Path -Parent (Get-Location).Path) 'ThothII-lf-repair'
if (Test-Path $RepairDir) { throw 'Choose a new empty LF repair directory.' }
New-Item -ItemType Directory -Path $RepairDir | Out-Null
$RepairPrefix = $RepairDir.Replace('\', '/') + '/'
git checkout-index --all --force --prefix=$RepairPrefix
git -c core.symlinks=true checkout-index --all --force --prefix=$RepairPrefix
Assert-NativeSuccess 'index export'
$RawIndexEntries = @(git ls-files -s)
Assert-NativeSuccess 'index inventory'
$IndexEntries = @($RawIndexEntries | ForEach-Object { ConvertFrom-IndexEntry $_ })
foreach ($Entry in $IndexEntries) {
$ExportPath = Join-Path $RepairDir $Entry.Path
$ExportItem = Get-Item -LiteralPath $ExportPath -Force -ErrorAction Stop
switch ($Entry.Mode) {
{ $_ -in '100644', '100755' } {
if ($ExportItem.LinkType -eq 'SymbolicLink') { throw "Regular export became a symlink: $($Entry.Path)" }
}
'120000' {
if ($ExportItem.LinkType -ne 'SymbolicLink') { throw "Symlink export is not mode 120000: $($Entry.Path)" }
if ([string]::IsNullOrWhiteSpace([string]$ExportItem.Target)) { throw "Symlink target is empty: $($Entry.Path)" }
}
default { throw "Unsupported Git mode $($Entry.Mode): $($Entry.Path)" }
}
}
& "C:\Program Files\Git\bin\bash.exe" scripts/verify-line-endings.sh $RepairDir
if ($LASTEXITCODE -ne 0) { throw 'The staged index export does not satisfy the LF policy.' }
Assert-NativeSuccess 'exported byte LF verification'
# WARNING: destructive copy; make a backup or commit wanted changes before this command.
git ls-files | ForEach-Object {
Copy-Item -LiteralPath (Join-Path $RepairDir $_) -Destination $_ -Force
foreach ($Entry in $IndexEntries) {
$ExportPath = Join-Path $RepairDir $Entry.Path
switch ($Entry.Mode) {
{ $_ -in '100644', '100755' } {
Copy-Item -LiteralPath $ExportPath -Destination $Entry.Path -Force -ErrorAction Stop
}
'120000' {
$LinkTarget = [string](Get-Item -LiteralPath $ExportPath -Force -ErrorAction Stop).Target
$TemporaryLink = "$($Entry.Path).thoth-lf-repair-link"
if (Test-Path -LiteralPath $TemporaryLink) { throw "Temporary symlink path exists: $TemporaryLink" }
New-Item -ItemType SymbolicLink -Path $TemporaryLink -Target $LinkTarget -ErrorAction Stop | Out-Null
Remove-Item -LiteralPath $Entry.Path -Force -ErrorAction Stop
Move-Item -LiteralPath $TemporaryLink -Destination $Entry.Path -ErrorAction Stop
}
default { throw "Unsupported Git mode $($Entry.Mode): $($Entry.Path)" }
}
}
foreach ($Entry in $IndexEntries) {
$WorktreeItem = Get-Item -LiteralPath $Entry.Path -Force -ErrorAction Stop
switch ($Entry.Mode) {
{ $_ -in '100644', '100755' } {
if ($WorktreeItem.LinkType -eq 'SymbolicLink') { throw "Regular worktree entry became a symlink: $($Entry.Path)" }
}
'120000' {
if ($WorktreeItem.LinkType -ne 'SymbolicLink') { throw "Repaired worktree symlink is not mode 120000: $($Entry.Path)" }
if ([string]::IsNullOrWhiteSpace([string]$WorktreeItem.Target)) { throw "Repaired symlink target is empty: $($Entry.Path)" }
}
default { throw "Unsupported Git mode $($Entry.Mode): $($Entry.Path)" }
}
}
& "C:\Program Files\Git\bin\bash.exe" scripts/verify-line-endings.sh
if ($LASTEXITCODE -ne 0) { throw 'Tracked worktree bytes were not repaired to the LF policy.' }
Assert-NativeSuccess 'repaired worktree LF verification'
git diff --cached --check
Assert-NativeSuccess 'repaired index check'
```
The first verifier proves the exported index bytes before any overwrite; the final verifier
examines the repaired worktree bytes and must also exit `0`. Review the staged diff again before
committing, then remove the separate repair directory only after inspecting it. The procedure
intentionally avoids `git reset --hard`; replacing the clone is easier to audit and much safer for
uncommitted work.
The export inventory must contain every regular mode (`100644`/`100755`) and recreate every tracked
workspace compatibility symlink (`120000`). The first verifier proves the complete
export before any overwrite; every copy/link operation is fail-closed; the final verifier examines
the repaired worktree bytes. On native Windows, creating symlinks requires Developer Mode or an
elevated account; failure stops the rewrite. Review the staged diff again before committing, then
remove the separate repair directory only after inspecting it. The procedure intentionally avoids
`git reset --hard`; replacing the clone is easier to audit and safer for uncommitted work.