From 5f015a5a3729e35a63a0999983eb021e15b832e6 Mon Sep 17 00:00:00 2001 From: mptyl Date: Wed, 5 Aug 2026 14:35:25 +0200 Subject: [PATCH] fix: complete deployment release gates --- .github/workflows/deployment.yml | 43 +- PROJECT_STATE.md | 41 +- README.md | 53 ++- scripts/server-deployment-smoke.sh | 8 + scripts/test-windows-clone-contract.ps1 | 313 ++++++++++--- scripts/thothctl-update-smoke.sh | 2 +- scripts/unified-deployment-smoke.sh | 513 +++++++++++++++++++++- tools/thothctl/cmd/thothctl/main.go | 7 + tools/thothctl/internal/pi/update.go | 8 +- tools/thothctl/internal/pi/update_test.go | 24 + 10 files changed, 891 insertions(+), 121 deletions(-) create mode 100755 scripts/server-deployment-smoke.sh diff --git a/.github/workflows/deployment.yml b/.github/workflows/deployment.yml index 85ee733c..96f8a674 100644 --- a/.github/workflows/deployment.yml +++ b/.github/workflows/deployment.yml @@ -5,6 +5,12 @@ on: push: branches: [main] workflow_dispatch: + inputs: + windows_docker_startup: + description: Run the native self-hosted Windows Docker Desktop/WSL2 release gate + required: false + type: boolean + default: false permissions: contents: read @@ -60,16 +66,18 @@ jobs: linux-docker: name: Linux Docker deployment and rollback runs-on: ubuntu-24.04 - timeout-minutes: 70 + timeout-minutes: 100 steps: - name: Check out source uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - name: Run unified deployment smoke - run: timeout --signal=TERM --kill-after=45s 30m bash scripts/unified-deployment-smoke.sh + run: timeout --signal=TERM --kill-after=45s 32m bash scripts/unified-deployment-smoke.sh - name: Run thothctl update smoke - run: timeout --signal=TERM --kill-after=45s 30m bash scripts/thothctl-update-smoke.sh + run: timeout --signal=TERM --kill-after=45s 32m bash scripts/thothctl-update-smoke.sh + - name: Run Linux server deployment smoke + run: timeout --signal=TERM --kill-after=45s 32m bash scripts/server-deployment-smoke.sh windows-clone: name: Windows clone and Compose contract @@ -85,14 +93,25 @@ jobs: with: go-version: "1.26.5" cache-dependency-path: tools/thothctl/go.sum - - name: Build native Windows thothctl - working-directory: tools/thothctl - shell: pwsh - run: | - New-Item -ItemType Directory -Force -Path ../../dist/thothctl | Out-Null - go build -trimpath -o ../../dist/thothctl/thothctl-windows-amd64.exe ./cmd/thothctl - name: Verify Windows clone contract shell: pwsh - run: >- - ./scripts/test-windows-clone-contract.ps1 - -ThothctlPath "$PWD/dist/thothctl/thothctl-windows-amd64.exe" + run: ./scripts/test-windows-clone-contract.ps1 + + windows-docker-release: + name: Native Windows Docker Desktop/WSL2 startup + if: github.event_name == 'workflow_dispatch' && inputs.windows_docker_startup + runs-on: [self-hosted, Windows, X64, docker-desktop] + timeout-minutes: 45 + steps: + - name: Check out source + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - name: Set up Go + uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 + with: + go-version: "1.26.5" + cache-dependency-path: tools/thothctl/go.sum + - name: Run spaced-path Windows Docker release gate + shell: pwsh + run: ./scripts/test-windows-clone-contract.ps1 -DockerStartup diff --git a/PROJECT_STATE.md b/PROJECT_STATE.md index d1987914..1a7dc859 100644 --- a/PROJECT_STATE.md +++ b/PROJECT_STATE.md @@ -9,28 +9,37 @@ frontend-to-core routing, embedded pinned Pi, Git registry bootstrap, offline recreation, valid update, invalid-update retention, and the four persistent stores. `scripts/thothctl-update-smoke.sh` independently exercises the bad-Pi update and automatic rollback path. + `scripts/server-deployment-smoke.sh` starts the server plus required session overlays with the + same smoke-built core/frontend images, disposable bind roots/secrets/session configuration, + upstream-auth checks, and fail-closed unavailable-session behavior. - **Isolation and disclosure boundary.** Every run generates a unique temporary root, Compose project, container/image names, transaction image tags, and run label. The rollback fixture uses - an immutable public digest as a deliberately dead core rather than a host-local image registry. - Cleanup checks ownership before removing exact containers, Compose resources, image references, - control state, and temporary files. There is no global prune. Failure diagnostics are bounded - and sanitized, and all credentials/endpoints used by the smokes are disposable fixtures rather - than operator or repository secrets. + an immutable `hello-world` digest whose preflight exits successfully, guaranteeing the stopped + core state required by `thothctl` compensation. Cleanup includes stopped project containers in + its final ownership check immediately before teardown and removes only exact containers, + Compose resources, image references, control state, and temporary files. There is no global + prune. Failure diagnostics are bounded and sanitized, and all credentials/endpoints used by the + smokes are disposable fixtures rather than operator or repository secrets. Every public smoke + also has an internal 30-minute process-group supervisor with TERM/KILL of the complete group. - **Cross-platform CI contract.** `.github/workflows/deployment.yml` uses immutable action commits, pinned supported Node and Go versions, runs LF/Compose/secret/coupling/docs/TypeScript gates on - Linux, runs each Docker smoke once under its own outer timeout, and builds/invokes native Windows - `thothctl` after the PowerShell clone/LF/Compose contract. Native Windows execution remains an - explicit manual release gate in addition to CI; no Windows Docker container startup is claimed - by the static clone job. + Linux, runs each Linux Docker smoke once under its own outer timeout, and copies the Windows + source into a path containing spaces before building/invoking native `thothctl` and rendering + Compose. The optional `windows_docker_startup` dispatch targets a labelled self-hosted Windows + Docker Desktop/WSL2 runner and performs bounded two-service startup and exact cleanup. No local + Windows or Windows Docker execution is claimed until that manual job is recorded. - **Validation status.** Deterministic Phase A gates, backend **434/434** plus TypeScript, frontend **386/386** plus TypeScript, and harness **862 passed / 5 L2 deselected** are green. - The unified one-shot Docker run passed frontend/core/internal Pi, registry bootstrap, offline - recreation, valid update, invalid-update retention, and persistence before Docker Desktop - refused the daemon-to-host local-registry push; the update-only run reached the same boundary. - Both exact run/project resource sets were independently proved absent. The local-registry - fixture was then removed in favor of the immutable dead-core digest, but the requested no-retry - rule leaves automatic rollback/preservation pending in CI or a fresh manual release run. Native - Windows PowerShell execution is also still a manual release gate. + Review round 1 ran each Docker smoke exactly once without retry. Unified (`103.86s`) and + update-only (`46.45s`) passed build/start, core/Pi/registry/persistence setup and the stopped + candidate preflight, but `thothctl` stopped before mutation at its active-session inventory gate. + A test-first fix now scopes local inventory to `mine` and supplies the fixture's missing direct + DWH/vector/embedding runtime bindings; the final rollback path was not rerun, so compensation + and all-sentinel preservation remain unproven. Server-profile execution (`12.88s`) built both + images but Docker Desktop/VirtioFS rejected the real profile's parent Pi-state bind plus nested + tracked agent-file binds before service startup. Every run's exact labelled cleanup passed. + Native-Linux server startup and native Windows PowerShell/Docker execution remain explicit + CI/manual release gates; no local success is claimed for either platform. ## Portable deployment decoupling — LIVE 2026-08-05 diff --git a/README.md b/README.md index 74c02553..68702c12 100644 --- a/README.md +++ b/README.md @@ -91,42 +91,67 @@ later with `docker compose --project-name "$SMOKE_PROJECT" down --volumes`. ## Unified deployment release gates -Task 13 adds a no-secret release gate around the canonical base plus local Compose profile. Its +Task 13 adds no-secret release gates around the canonical local and server Compose profiles. Its deterministic safety check does not contact the Docker daemon: ```sh bash scripts/unified-deployment-smoke.sh --self-test ``` -The two Docker smokes are separate release jobs. Each creates a unique Compose project, temporary +The three Linux Docker smokes are separate release commands. Each creates a unique Compose project, temporary Git workspace remote, fixture provider, image names, and run label. Its exit trap removes only -resources carrying that exact run identity and never performs a global Docker prune. +resources carrying that exact run identity and never performs a global Docker prune. Cleanup +enumerates running and stopped project containers immediately before `compose down` and refuses +the teardown if any container, volume, or network has a foreign run label. ```sh bash scripts/unified-deployment-smoke.sh bash scripts/thothctl-update-smoke.sh +bash scripts/server-deployment-smoke.sh ``` The unified smoke builds and starts `frontend` and `core`, verifies the embedded Pi and internal registry, recreates with the Git remote offline, activates a valid Git update, rejects invalid Git -content while retaining the valid snapshot, and checks the four persistence volumes. Both smokes +content while retaining the valid snapshot, and checks the four persistence volumes. The unified +and update-only smokes inject a digest-pinned non-core candidate under a deliberately mismatched Pi version and require `thothctl pi update` to roll back while preserving settings, sessions, Pi state, registry revision, -and mount identity. Fixture credentials are generated locally; neither command needs a real -provider key or a repository secret. CI gives each smoke one 30-minute outer timeout and does not -retry it. +and mount identity. The rollback candidate is the digest-pinned `hello-world` executable: a +preflight proves that it exits successfully, so the failed replacement core satisfies +`thothctl`'s stopped-core compensation precondition. The server smoke uses the same smoke-built +core/frontend images with the server and required session overlays, disposable bind roots and +secret files, upstream-auth checks, and a fail-closed `503` assertion for its deliberately +unavailable disposable session endpoint. No real provider, database credential, or repository +secret is required. -On a native Windows clone, the release contract is: +Each public smoke has its own 30-minute process-group supervisor with TERM/KILL cleanup; CI retains +an independent 32-minute outer timeout and does not retry a failed command. + +Current release status (2026-08-05): deterministic contracts are green, but the complete rollback +fixture has not passed end to end after its runtime-binding correction. The one observed local +server-profile run also stopped before startup because Docker Desktop/VirtioFS rejected the +profile's parent Pi-state bind with nested tracked agent-file binds. A fresh single rollback run, +native-Linux server-profile run, and native Windows Docker Desktop/WSL2 run remain release gates; +the project does not claim those criteria green. + +The deterministic native Windows contract is: ```powershell -.\scripts\test-windows-clone-contract.ps1 ` - -ThothctlPath "$PWD\dist\thothctl\thothctl-windows-amd64.exe" +.\scripts\test-windows-clone-contract.ps1 ``` -It checks Git's CRLF/LF attributes and bytes, renders exactly `core` plus `frontend` with Docker -Compose without starting containers, and invokes the native Windows `thothctl`. The GitHub Actions -deployment workflow runs the deterministic Linux gates, both bounded Docker smokes, and this -Windows clone contract with immutable action pins and supported pinned Node/Go toolchains. +It checks Git's CRLF/LF attributes and bytes, copies tracked source into a temporary path containing +spaces, builds and invokes native Windows `thothctl` there, and renders exactly `core` plus +`frontend` without starting containers. On a supported self-hosted Windows Docker Desktop/WSL2 +runner, dispatch the deployment workflow with `windows_docker_startup=true`; that job executes: + +```powershell +.\scripts\test-windows-clone-contract.ps1 -DockerStartup +``` + +Startup mode adds bounded image build/two-service health startup, installation-aware `thothctl` +status, stopped-container-aware ownership checks, and exact cleanup. The ordinary hosted Windows +job remains deterministic and does not claim Docker startup. ## Optional local pgvector and recovery diff --git a/scripts/server-deployment-smoke.sh b/scripts/server-deployment-smoke.sh new file mode 100755 index 00000000..f0d7c203 --- /dev/null +++ b/scripts/server-deployment-smoke.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -euo pipefail + +root="$(cd "$(dirname "$0")/.." && pwd -P)" +# shellcheck source=./unified-deployment-smoke.sh +source "$root/scripts/unified-deployment-smoke.sh" + +task13_supervise "$TASK13_SMOKE_TIMEOUT" "Linux server deployment smoke" task13_server_smoke_main diff --git a/scripts/test-windows-clone-contract.ps1 b/scripts/test-windows-clone-contract.ps1 index 41354b0c..12e77312 100644 --- a/scripts/test-windows-clone-contract.ps1 +++ b/scripts/test-windows-clone-contract.ps1 @@ -1,27 +1,90 @@ param( [string]$RepositoryRoot = "", - [string]$ThothctlPath = "" + [switch]$DockerStartup, + [int]$CommandTimeoutSeconds = 600 ) $ErrorActionPreference = "Stop" Set-StrictMode -Version Latest +$script:SensitiveValues = [System.Collections.Generic.List[string]]::new() + +function Protect-Output([string]$Value) { + $protected = $Value + foreach ($secret in $script:SensitiveValues) { + if (-not [string]::IsNullOrEmpty($secret)) { + $protected = $protected.Replace($secret, "[REDACTED]") + } + } + return $protected -replace '(?i)((?:password|token|api[_-]?key|secret|key)\s*[:=]\s*)[^\s,;]+', '$1[REDACTED]' +} + +function Invoke-BoundedNative { + param( + [Parameter(Mandatory = $true)][string]$FilePath, + [Parameter(Mandatory = $true)][string[]]$Arguments, + [Parameter(Mandatory = $true)][string]$Label, + [string]$WorkingDirectory = "", + [int]$TimeoutSeconds = $CommandTimeoutSeconds, + [switch]$AllowFailure + ) + $startInfo = [System.Diagnostics.ProcessStartInfo]::new() + $startInfo.FileName = $FilePath + $startInfo.UseShellExecute = $false + $startInfo.RedirectStandardOutput = $true + $startInfo.RedirectStandardError = $true + $startInfo.CreateNoWindow = $true + if (-not [string]::IsNullOrWhiteSpace($WorkingDirectory)) { + $startInfo.WorkingDirectory = $WorkingDirectory + } + foreach ($argument in $Arguments) { + [void]$startInfo.ArgumentList.Add($argument) + } + $process = [System.Diagnostics.Process]::new() + $process.StartInfo = $startInfo + if (-not $process.Start()) { + throw "$Label could not start" + } + $stdoutTask = $process.StandardOutput.ReadToEndAsync() + $stderrTask = $process.StandardError.ReadToEndAsync() + if (-not $process.WaitForExit($TimeoutSeconds * 1000)) { + $process.Kill($true) + [void]$process.WaitForExit(30000) + throw "$Label timed out after $TimeoutSeconds seconds" + } + $stdout = $stdoutTask.GetAwaiter().GetResult() + $stderr = $stderrTask.GetAwaiter().GetResult() + $result = [pscustomobject]@{ + ExitCode = $process.ExitCode + StdOut = $stdout + StdErr = $stderr + } + if (-not $AllowFailure -and $result.ExitCode -ne 0) { + $diagnostic = Protect-Output (($result.StdOut + "`n" + $result.StdErr).Trim()) + throw "$Label failed with exit $($result.ExitCode): $diagnostic" + } + return $result +} + +function Write-Utf8File([string]$Path, [string]$Contents) { + [System.IO.Directory]::CreateDirectory([System.IO.Path]::GetDirectoryName($Path)) | Out-Null + [System.IO.File]::WriteAllText($Path, $Contents, [System.Text.UTF8Encoding]::new($false)) +} + +function ConvertTo-YamlPath([string]$Path) { + return $Path.Replace('\', '/').Replace('"', '\"') +} if ([string]::IsNullOrWhiteSpace($RepositoryRoot)) { - $RepositoryRoot = (& git rev-parse --show-toplevel).Trim() - if ($LASTEXITCODE -ne 0) { - throw "git could not resolve the repository root" - } + $resolved = Invoke-BoundedNative -FilePath "git" -Arguments @("rev-parse", "--show-toplevel") -Label "resolve repository root" + $RepositoryRoot = $resolved.StdOut.Trim() } $RepositoryRoot = [System.IO.Path]::GetFullPath($RepositoryRoot) -$tracked = @(& git -C $RepositoryRoot ls-files) -if ($LASTEXITCODE -ne 0) { - throw "git ls-files failed" -} - +$trackedResult = Invoke-BoundedNative -FilePath "git" -Arguments @("-C", $RepositoryRoot, "ls-files") -Label "list tracked files" +$tracked = @($trackedResult.StdOut -split "`r?`n" | Where-Object { -not [string]::IsNullOrWhiteSpace($_) }) $scriptRelativePath = "scripts/test-windows-clone-contract.ps1" -$eolAttribute = (& git -C $RepositoryRoot check-attr eol -- $scriptRelativePath).Trim() -if ($LASTEXITCODE -ne 0 -or -not $eolAttribute.EndsWith("eol: crlf", [System.StringComparison]::OrdinalIgnoreCase)) { +$attribute = Invoke-BoundedNative -FilePath "git" -Arguments @("-C", $RepositoryRoot, "check-attr", "eol", "--", $scriptRelativePath) -Label "read PowerShell eol attribute" +if (-not $attribute.StdOut.Trim().EndsWith("eol: crlf", [System.StringComparison]::OrdinalIgnoreCase)) { throw "the Windows contract script must have the repository eol=crlf attribute" } $scriptBytes = [System.IO.File]::ReadAllBytes((Join-Path $RepositoryRoot $scriptRelativePath)) @@ -38,71 +101,203 @@ foreach ($relativePath in $tracked) { $name.Equals("Dockerfile", [System.StringComparison]::OrdinalIgnoreCase) -or $name.StartsWith("Dockerfile.", [System.StringComparison]::OrdinalIgnoreCase) -or $name.EndsWith(".Dockerfile", [System.StringComparison]::OrdinalIgnoreCase) - if (-not $mustBeLf) { - continue - } - $absolutePath = Join-Path $RepositoryRoot $relativePath - $bytes = [System.IO.File]::ReadAllBytes($absolutePath) - if ($bytes -contains [byte]0x0D) { - $offenders.Add($relativePath) + if ($mustBeLf) { + $bytes = [System.IO.File]::ReadAllBytes((Join-Path $RepositoryRoot $relativePath)) + if ($bytes -contains [byte]0x0D) { + $offenders.Add($relativePath) + } } } if ($offenders.Count -ne 0) { throw "CR byte 0x0D found in tracked LF contract files: $($offenders -join ', ')" } -$temporaryRoot = Join-Path ([System.IO.Path]::GetTempPath()) ("thothii-windows-contract-" + [guid]::NewGuid().ToString("N")) -$savedEnvironment = @{ - THT_WORKSPACE_GIT_REMOTE = $env:THT_WORKSPACE_GIT_REMOTE - PI_AUTH_FILE = $env:PI_AUTH_FILE - THT_SECRETS_FILE = $env:THT_SECRETS_FILE -} -try { - [System.IO.Directory]::CreateDirectory($temporaryRoot) | Out-Null - $piAuth = Join-Path $temporaryRoot "pi-auth.json" - $secrets = Join-Path $temporaryRoot "thothii.secrets" - [System.IO.File]::WriteAllText($piAuth, "{}`n", [System.Text.UTF8Encoding]::new($false)) - [System.IO.File]::WriteAllText($secrets, "THT_MODEL_API_KEY=windows-contract`n", [System.Text.UTF8Encoding]::new($false)) +$runId = [guid]::NewGuid().ToString("N") +$temporaryRoot = Join-Path ([System.IO.Path]::GetTempPath()) ("ThothII Task 13 path with spaces " + $runId) +$spacedRepository = Join-Path $temporaryRoot "Task 13 path with spaces" +$fixtureRoot = Join-Path $temporaryRoot "Disposable Fixture Data" +$project = "thothii-win-" + $runId.Substring(0, 12) +$runLabel = "windows-" + $runId +$coreImage = "task13-windows-core-$($runId.Substring(0, 16)):local" +$frontendImage = "task13-windows-frontend-$($runId.Substring(0, 16)):local" +$composeArguments = @() +$startupAttempted = $false +$cleanupSucceeded = $true +$savedEnvironment = @{} - $env:THT_WORKSPACE_GIT_REMOTE = "https://git.example.invalid/platform/thoth-workspaces.git" - $env:PI_AUTH_FILE = $piAuth - $env:THT_SECRETS_FILE = $secrets - $composeFiles = @( - "--project-directory", $RepositoryRoot, - "-f", (Join-Path $RepositoryRoot "compose.yaml"), - "-f", (Join-Path $RepositoryRoot "deploy/compose.local.yaml") - ) - $services = @(& docker compose @composeFiles config --services) - if ($LASTEXITCODE -ne 0) { - throw "Docker Compose could not render the Windows clone" +try { + [System.IO.Directory]::CreateDirectory($spacedRepository) | Out-Null + foreach ($relativePath in $tracked) { + $source = Join-Path $RepositoryRoot $relativePath + $destination = Join-Path $spacedRepository $relativePath + [System.IO.Directory]::CreateDirectory([System.IO.Path]::GetDirectoryName($destination)) | Out-Null + Copy-Item -LiteralPath $source -Destination $destination } - if ((($services | Sort-Object) -join ",") -ne "core,frontend") { + + $thothctl = Join-Path $spacedRepository "dist/thothctl/thothctl-windows-amd64.exe" + [System.IO.Directory]::CreateDirectory([System.IO.Path]::GetDirectoryName($thothctl)) | Out-Null + Invoke-BoundedNative -FilePath "go" -Arguments @("build", "-trimpath", "-o", $thothctl, "./cmd/thothctl") ` + -WorkingDirectory (Join-Path $spacedRepository "tools/thothctl") -Label "build native Windows thothctl in spaced path" | Out-Null + Invoke-BoundedNative -FilePath $thothctl -Arguments @("--help") -Label "invoke native Windows thothctl from spaced path" | Out-Null + + $piAuth = Join-Path $fixtureRoot "Pi Auth/pi-auth.json" + $secrets = Join-Path $fixtureRoot "Secrets/thothii.secrets" + $secretValue = "windows-contract-$runId" + $script:SensitiveValues.Add($secretValue) + Write-Utf8File $piAuth "{}`n" + Write-Utf8File $secrets "THT_MODEL_API_KEY=$secretValue`n" + + $remote = Join-Path $fixtureRoot "Workspace Remote/remote.git" + $seed = Join-Path $fixtureRoot "Workspace Seed" + [System.IO.Directory]::CreateDirectory((Join-Path $seed "workspaces")) | Out-Null + Invoke-BoundedNative -FilePath "git" -Arguments @("init", "--bare", "--initial-branch=main", $remote) -Label "initialize Windows bare registry" | Out-Null + Invoke-BoundedNative -FilePath "git" -Arguments @("-C", $seed, "init", "--initial-branch=main") -Label "initialize Windows registry seed" | Out-Null + Write-Utf8File (Join-Path $seed "workspaces/task13-windows.yaml") @" +workspace: + schema_version: 2 + id: task13-windows + name: Task 13 Windows + language: en +dwh: + engine: postgres + database: warehouse + schema: public + supported_transports: [postgres_direct] +"@ + Invoke-BoundedNative -FilePath "git" -Arguments @("-C", $seed, "add", "workspaces/task13-windows.yaml") -Label "stage Windows registry seed" | Out-Null + Invoke-BoundedNative -FilePath "git" -Arguments @("-C", $seed, "-c", "user.name=Task 13 Windows", "-c", "user.email=task13-windows@example.invalid", "commit", "-m", "Seed Windows smoke") -Label "commit Windows registry seed" | Out-Null + Invoke-BoundedNative -FilePath "git" -Arguments @("-C", $seed, "push", $remote, "HEAD:main") -Label "push Windows registry seed" | Out-Null + + $envFile = Join-Path $fixtureRoot "Config/local.env" + $override = Join-Path $fixtureRoot "Config/compose.windows.yaml" + $installation = Join-Path $fixtureRoot "Config/thothii installation.yaml" + Write-Utf8File $envFile @" +THOTH_HTTP_PORT=0 +THOTH_CORE_HTTP_PORT=0 +PI_AUTH_FILE=$piAuth +THT_SECRETS_FILE=$secrets +THT_WORKSPACE_GIT_REMOTE=/fixtures/remote.git +THT_WORKSPACE_GIT_BRANCH=main +THT_WORKSPACE_INSTALLATION_ID=task13-windows +"@ + $remoteYaml = ConvertTo-YamlPath $remote + Write-Utf8File $override @" +services: + core: + image: $coreImage + build: + labels: + io.thothii.task13.run: "$runLabel" + labels: + io.thothii.task13.run: "$runLabel" + volumes: + - "$remoteYaml:/fixtures/remote.git:ro" + frontend: + image: $frontendImage + build: + labels: + io.thothii.task13.run: "$runLabel" + labels: + io.thothii.task13.run: "$runLabel" +networks: + thothii: + labels: + io.thothii.task13.run: "$runLabel" +volumes: + settings: + labels: + io.thothii.task13.run: "$runLabel" + pi-state: + labels: + io.thothii.task13.run: "$runLabel" + workspace-registry: + labels: + io.thothii.task13.run: "$runLabel" + sessions: + labels: + io.thothii.task13.run: "$runLabel" +"@ + $repoYaml = ConvertTo-YamlPath $spacedRepository + $envYaml = ConvertTo-YamlPath $envFile + $overrideYaml = ConvertTo-YamlPath $override + Write-Utf8File $installation @" +profile: local +projectDirectory: "$repoYaml" +envFile: "$envYaml" +overrides: + - "$overrideYaml" +"@ + + $composeArguments = @( + "compose", "--project-name", $project, "--project-directory", $spacedRepository, + "--env-file", $envFile, + "-f", (Join-Path $spacedRepository "compose.yaml"), + "-f", (Join-Path $spacedRepository "deploy/compose.local.yaml"), + "-f", $override + ) + $render = Invoke-BoundedNative -FilePath "docker" -Arguments ($composeArguments + @("config", "--services")) -Label "render Windows Compose from spaced path" + $services = @($render.StdOut -split "`r?`n" | Where-Object { $_ } | Sort-Object) + if (($services -join ",") -ne "core,frontend") { throw "rendered Windows stack must contain exactly core and frontend" } - & docker compose @composeFiles config --quiet - if ($LASTEXITCODE -ne 0) { - throw "Docker Compose rejected the Windows clone" - } + Invoke-BoundedNative -FilePath "docker" -Arguments ($composeArguments + @("config", "--quiet")) -Label "validate Windows Compose from spaced path" | Out-Null - if ([string]::IsNullOrWhiteSpace($ThothctlPath)) { - $ThothctlPath = Join-Path $RepositoryRoot "dist/thothctl/thothctl-windows-amd64.exe" - } - $ThothctlPath = [System.IO.Path]::GetFullPath($ThothctlPath) - if (-not [System.IO.File]::Exists($ThothctlPath)) { - throw "Windows thothctl binary is missing: $ThothctlPath" - } - & $ThothctlPath --help | Out-Null - if ($LASTEXITCODE -ne 0) { - throw "Windows thothctl invocation failed" + if ($DockerStartup) { + Invoke-BoundedNative -FilePath "docker" -Arguments @("info") -Label "verify Windows Docker Desktop readiness" -TimeoutSeconds 60 | Out-Null + $startupAttempted = $true + Invoke-BoundedNative -FilePath "docker" -Arguments ($composeArguments + @("build", "--pull")) -Label "build two-service Windows stack" | Out-Null + Invoke-BoundedNative -FilePath "docker" -Arguments ($composeArguments + @("up", "--detach", "--wait", "--wait-timeout", "180")) -Label "start two-service Windows stack" -TimeoutSeconds 300 | Out-Null + $running = Invoke-BoundedNative -FilePath "docker" -Arguments ($composeArguments + @("ps", "--status", "running", "--services")) -Label "inspect running Windows services" + $runningServices = @($running.StdOut -split "`r?`n" | Where-Object { $_ } | Sort-Object) + if (($runningServices -join ",") -ne "core,frontend") { + throw "bounded Windows startup did not leave exactly core and frontend running" + } + Invoke-BoundedNative -FilePath $thothctl -Arguments @("--installation", $installation, "status") -Label "invoke installation-aware Windows thothctl in spaced path" | Out-Null } } finally { + if ($startupAttempted -and $composeArguments.Count -ne 0) { + try { + foreach ($kind in @("container", "volume", "network")) { + $listArgs = if ($kind -eq "container") { @($kind, "ls", "-aq") } else { @($kind, "ls", "-q") } + $listed = Invoke-BoundedNative -FilePath "docker" -Arguments ($listArgs + @("--filter", "label=com.docker.compose.project=$project")) -Label "enumerate Windows project $kind resources" -TimeoutSeconds 30 + foreach ($id in @($listed.StdOut -split "`r?`n" | Where-Object { $_ })) { + $format = if ($kind -eq "container") { '{{ index .Config.Labels "io.thothii.task13.run" }}' } else { '{{ index .Labels "io.thothii.task13.run" }}' } + $inspected = Invoke-BoundedNative -FilePath "docker" -Arguments @($kind, "inspect", "--format", $format, $id) -Label "inspect Windows project $kind ownership" -TimeoutSeconds 30 + if ($inspected.StdOut.Trim() -ne $runLabel) { + throw "refusing to remove foreign Windows project $kind resource" + } + } + } + Invoke-BoundedNative -FilePath "docker" -Arguments ($composeArguments + @("down", "--volumes", "--remove-orphans", "--timeout", "10")) -Label "remove exact Windows Compose project" -TimeoutSeconds 60 | Out-Null + foreach ($image in @($frontendImage, $coreImage)) { + $inspection = Invoke-BoundedNative -FilePath "docker" -Arguments @("image", "inspect", "--format", '{{ index .Config.Labels "io.thothii.task13.run" }}', $image) -Label "inspect Windows image ownership" -TimeoutSeconds 30 -AllowFailure + if ($inspection.ExitCode -eq 0) { + if ($inspection.StdOut.Trim() -ne $runLabel) { + throw "refusing to remove foreign Windows image $image" + } + Invoke-BoundedNative -FilePath "docker" -Arguments @("image", "rm", $image) -Label "remove exact Windows image" -TimeoutSeconds 60 | Out-Null + } + } + } + catch { + $cleanupSucceeded = $false + Write-Error (Protect-Output $_.Exception.Message) + } + } foreach ($name in $savedEnvironment.Keys) { [System.Environment]::SetEnvironmentVariable($name, $savedEnvironment[$name], "Process") } - if ([System.IO.Directory]::Exists($temporaryRoot)) { + if ($cleanupSucceeded -and [System.IO.Directory]::Exists($temporaryRoot)) { Remove-Item -LiteralPath $temporaryRoot -Recurse -Force } } -Write-Output "Windows clone, LF-byte, Compose render, and thothctl invocation contracts passed." +if (-not $cleanupSucceeded) { + throw "Windows cleanup proof failed; fixture path retained for recovery" +} +if ($DockerStartup) { + Write-Output "Windows spaced-path build, native thothctl, bounded two-service startup, and exact cleanup passed." +} else { + Write-Output "Windows spaced-path clone, LF-byte, Compose render, and native thothctl build/invocation contracts passed; Docker startup mode was not requested." +} diff --git a/scripts/thothctl-update-smoke.sh b/scripts/thothctl-update-smoke.sh index 5059306b..b6c3f6c5 100755 --- a/scripts/thothctl-update-smoke.sh +++ b/scripts/thothctl-update-smoke.sh @@ -5,4 +5,4 @@ root="$(cd "$(dirname "$0")/.." && pwd -P)" # shellcheck source=./unified-deployment-smoke.sh source "$root/scripts/unified-deployment-smoke.sh" -task13_smoke_main update +task13_supervise "$TASK13_SMOKE_TIMEOUT" "thothctl update smoke" task13_smoke_main update diff --git a/scripts/unified-deployment-smoke.sh b/scripts/unified-deployment-smoke.sh index 62e95a65..23c488db 100755 --- a/scripts/unified-deployment-smoke.sh +++ b/scripts/unified-deployment-smoke.sh @@ -4,11 +4,15 @@ # isolated fixture, exact cleanup, and sanitized failure reporting. set -euo pipefail -TASK13_BAD_CANDIDATE_IMAGE="registry:2.8.3@sha256:a3d8aaa63ed8681a604f1dea0aa03f100d5895b6a58ace528858a7b332415373" +TASK13_BAD_CANDIDATE_IMAGE="hello-world@sha256:c3cbe1cc1aa588a64951ac6286e0df7b27fe2e6324b1001c619bb358770c0178" +TASK13_BAD_CANDIDATE_BEHAVIOR="stopped" TASK13_BAD_PI_VERSION="0.80.4-task13" TASK13_CURL_CONNECT_TIMEOUT=3 TASK13_CURL_MAX_TIME=10 TASK13_CLEANUP_TIMEOUT=20 +TASK13_COMMAND_TIMEOUT=900 +TASK13_SMOKE_TIMEOUT=1800 +TASK13_TERM_GRACE=45 task13_fail() { printf 'Task 13 smoke failed: %s\n' "$*" >&2 @@ -49,41 +53,82 @@ task13_log_failure() { task13_run_logged() { local label="$1" shift - if ! "$@" >>"$TASK13_LOG" 2>&1; then + if ! task13_bounded "$TASK13_COMMAND_TIMEOUT" "$label" "$@" >>"$TASK13_LOG" 2>&1; then task13_log_failure "$label" fi } task13_bounded() { - local seconds="$1" label="$2" command_pid watchdog_pid rc + local seconds="$1" label="$2" command_pid watchdog_pid rc watchdog_rc=0 + local monitor_enabled=0 timeout_marker command_group shift 2 - "$@" & - command_pid=$! + timeout_marker="$(mktemp "${TMPDIR:-/tmp}/thothii-task13-timeout.XXXXXX")" + [[ $- == *m* ]] && monitor_enabled=1 + if [[ -n "${TASK13_ACTIVE_GROUP:-}" ]]; then + [[ "$TASK13_ACTIVE_GROUP" =~ ^[0-9]+$ ]] \ + || task13_fail "invalid active Task 13 process group" + set +m + "$@" & + command_pid=$! + command_group="$TASK13_ACTIVE_GROUP" + else + set -m + "$@" & + command_pid=$! + command_group="$command_pid" + [[ "$monitor_enabled" -eq 1 ]] || set +m + fi ( - local sleep_pid + local sleep_pid grace_deadline sleep "$seconds" & sleep_pid=$! trap 'kill "$sleep_pid" 2>/dev/null || true' EXIT INT TERM wait "$sleep_pid" 2>/dev/null || exit 0 trap - EXIT INT TERM - if kill -0 "$command_pid" 2>/dev/null; then + if kill -0 -- "-$command_group" 2>/dev/null; then + trap '' TERM + printf 'timeout\n' >"$timeout_marker" printf 'Task 13 command timed out after %ss: %s\n' "$seconds" "$label" >&2 - kill -TERM "$command_pid" 2>/dev/null || true - sleep 5 - kill -KILL "$command_pid" 2>/dev/null || true + kill -TERM -- "-$command_group" 2>/dev/null || true + grace_deadline=$((SECONDS + TASK13_TERM_GRACE)) + while kill -0 -- "-$command_group" 2>/dev/null && ((SECONDS < grace_deadline)); do + sleep 1 + done + kill -KILL -- "-$command_group" 2>/dev/null || true + exit 124 fi ) & watchdog_pid=$! + if [[ -n "${TASK13_ACTIVE_GROUP:-}" && "$monitor_enabled" -eq 1 ]]; then + set -m + fi if wait "$command_pid"; then rc=0 else rc=$? fi - kill "$watchdog_pid" 2>/dev/null || true - wait "$watchdog_pid" 2>/dev/null || true + if [[ -s "$timeout_marker" ]]; then + wait "$watchdog_pid" 2>/dev/null || watchdog_rc=$? + else + kill "$watchdog_pid" 2>/dev/null || true + wait "$watchdog_pid" 2>/dev/null || true + fi + rm -f "$timeout_marker" + [[ "$watchdog_rc" -eq 124 ]] && return 124 return "$rc" } +task13_supervised_call() { + TASK13_ACTIVE_GROUP="$BASHPID" + "$@" +} + +task13_supervise() { + local seconds="$1" label="$2" + shift 2 + task13_bounded "$seconds" "$label" task13_supervised_call "$@" +} + task13_compose_files() { TASK13_COMPOSE=( docker compose @@ -91,10 +136,20 @@ task13_compose_files() { --project-directory "$TASK13_ROOT" --env-file "$TASK13_ENV_FILE" -f "$TASK13_ROOT/compose.yaml" - -f "$TASK13_ROOT/deploy/compose.local.yaml" - -f "$TASK13_OVERRIDE" ) - if [[ -f "$TASK13_CURRENT_IMAGE_OVERRIDE" ]]; then + if [[ "${TASK13_PROFILE:-local}" == server ]]; then + TASK13_COMPOSE+=( + -f "$TASK13_ROOT/deploy/compose.server.yaml" + -f "$TASK13_ROOT/deploy/compose.session-server.yaml.example" + -f "$TASK13_OVERRIDE" + ) + else + TASK13_COMPOSE+=( + -f "$TASK13_ROOT/deploy/compose.local.yaml" + -f "$TASK13_OVERRIDE" + ) + fi + if [[ "${TASK13_PROFILE:-local}" == local && -f "$TASK13_CURRENT_IMAGE_OVERRIDE" ]]; then TASK13_COMPOSE+=(-f "$TASK13_CURRENT_IMAGE_OVERRIDE") fi } @@ -224,6 +279,15 @@ services: PI_THINKING: low THT_WORKSPACE_INSTALLATION_ID: task13-smoke THT_WORKSPACE_REGISTRY_ROOT: /data/workspace-registry + THT_WS_TASK13_SMOKE_DWH_HOST: dwh.task13.invalid + THT_WS_TASK13_SMOKE_DWH_PORT: "5432" + THT_WS_TASK13_SMOKE_DWH_USER: task13_reader + THT_WS_TASK13_SMOKE_DWH_PASSWORD_FILE: /run/secrets/thothii.secrets + THT_WS_TASK13_SMOKE_VECTOR_HOST: vector.task13.invalid + THT_WS_TASK13_SMOKE_VECTOR_PORT: "5432" + THT_WS_TASK13_SMOKE_VECTOR_USER: task13_vector_reader + THT_WS_TASK13_SMOKE_VECTOR_PASSWORD_FILE: /run/secrets/thothii.secrets + THT_WS_TASK13_SMOKE_EMBEDDING_BASE_URL: http://$TASK13_LLM_CONTAINER:9000 labels: io.thothii.task13.run: "$TASK13_RUN_ID" volumes: !override @@ -272,6 +336,114 @@ EOF chmod 0600 "$TASK13_INSTALLATION" } +task13_write_server_fixture_files() { + local data_root pi_root registry_root remote_path workspace_path + printf '{}\n' >"$TASK13_PI_AUTH" + printf 'THT_MODEL_API_KEY=%s\n' "$TASK13_SECRET_VALUE" >"$TASK13_SECRETS" + printf '%s\n' "$TASK13_SESSION_PASSWORD" >"$TASK13_SESSION_RUNTIME_PASSWORD" + printf '%s\n' "$TASK13_SESSION_MIGRATOR_PASSWORD" >"$TASK13_SESSION_MIGRATOR_PASSWORD_FILE" + cat >"$TASK13_SESSION_CA" <<'EOF' +-----BEGIN CERTIFICATE----- +VEFTSzEzLURJU1BPU0FCTEUtU0VTU0lPTi1DQQ== +-----END CERTIFICATE----- +EOF + chmod 0644 "$TASK13_PI_AUTH" "$TASK13_SESSION_CA" + chmod 0600 "$TASK13_SECRETS" "$TASK13_SESSION_RUNTIME_PASSWORD" \ + "$TASK13_SESSION_MIGRATOR_PASSWORD_FILE" + + cat >"$TASK13_SERVER_WORKSPACE_CONFIG" <<'EOF' +language: en +session_storage: + type: postgres_direct + connection: + host: ${THT_SESSION_DB_HOST} + port: ${THT_SESSION_DB_PORT} + database: ${THT_SESSION_DB_NAME} + schema: thoth_sessions + user: ${THT_SESSION_RUNTIME_USER} + password_file: ${THT_SESSION_RUNTIME_PASSWORD_FILE} + sslmode: ${THT_SESSION_DB_SSLMODE} + sslrootcert: ${THT_SESSION_DB_SSLROOTCERT} +roots: + artifacts: artifacts + indexes: indexes + sessions: sessions +EOF + chmod 0600 "$TASK13_SERVER_WORKSPACE_CONFIG" + + mkdir -p "$TASK13_SERVER_DATA" "$TASK13_SERVER_PI_STATE" "$TASK13_SERVER_REGISTRY" + chmod 0777 "$TASK13_SERVER_DATA" "$TASK13_SERVER_PI_STATE" "$TASK13_SERVER_REGISTRY" + data_root="$TASK13_SERVER_DATA" + pi_root="$TASK13_SERVER_PI_STATE" + registry_root="$TASK13_SERVER_REGISTRY" + remote_path="$TASK13_REMOTE" + workspace_path="$TASK13_SERVER_WORKSPACE_CONFIG" + + cat >"$TASK13_OVERRIDE" <"$TASK13_ENV_FILE" + chmod 0600 "$TASK13_ENV_FILE" +} + task13_seed_registry() { mkdir -p "$TASK13_SEED/workspaces" task13_run_logged "initialize bare workspace registry" \ @@ -377,6 +549,14 @@ task13_start_stack() { task13_log_failure "deterministic local LLM fixture readiness" } +task13_start_server_stack() { + printf '== Build and start isolated Linux server profile ==\n' + task13_assert_rendered_contract + task13_compose_logged "build server Compose images" build --pull core frontend + task13_compose_logged "start server Compose distribution" \ + up --detach --wait --wait-timeout 120 core frontend +} + task13_frontend_address() { task13_compose port frontend 8080 | awk 'NR == 1 {print $0}' } @@ -422,6 +602,67 @@ task13_assert_runtime() { task13_run_logged "thothctl Pi doctor" "$TASK13_THOTHCTL" --installation "$TASK13_INSTALLATION" pi doctor } +task13_assert_server_runtime() { + local frontend unauthenticated authenticated session_status core_id frontend_id + local expected_core_image expected_frontend_image + frontend="$(task13_frontend_address)" + task13_run_logged "server frontend health" curl \ + --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" --max-time "$TASK13_CURL_MAX_TIME" \ + --fail --silent --show-error "http://$frontend/" + task13_run_logged "server same-origin core health" curl \ + --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" --max-time "$TASK13_CURL_MAX_TIME" \ + --fail --silent --show-error "http://$frontend/api/health" + + core_id="$(task13_core_id)" + frontend_id="$(task13_compose ps -q frontend)" + expected_core_image="$(docker image inspect --format '{{.Id}}' "$TASK13_CORE_IMAGE")" + expected_frontend_image="$(docker image inspect --format '{{.Id}}' "$TASK13_FRONTEND_IMAGE")" + [[ "$(docker inspect --format '{{.Image}}' "$core_id")" == "$expected_core_image" ]] \ + || task13_fail "server core did not use the smoke-built core image" + [[ "$(docker inspect --format '{{.Image}}' "$frontend_id")" == "$expected_frontend_image" ]] \ + || task13_fail "server frontend did not use the smoke-built frontend image" + task13_compose exec -T core sh -ceu ' + test "$AUTH_MODE" = upstream + test "$THT_SESSION_STORAGE" = postgres + test -r /run/secrets/thothii.secrets + test -r /run/secrets/session_runtime_password + test -r /run/secrets/session_ca.pem + test -r /app/harness/workspaces/server-sessions.yaml + ' + task13_mount_fingerprint | grep -Fq '/data = bind :' \ + || task13_fail "server profile did not bind the disposable data root" + task13_mount_fingerprint | grep -Fq '/home/thoth/.pi = bind :' \ + || task13_fail "server profile did not bind the disposable Pi state root" + task13_mount_fingerprint | grep -Fq '/data/workspace-registry = bind :' \ + || task13_fail "server profile did not bind the disposable registry root" + + unauthenticated="$(curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" \ + --max-time "$TASK13_CURL_MAX_TIME" --silent --output /dev/null --write-out '%{http_code}' \ + "http://$frontend/api/workspaces")" + [[ "$unauthenticated" == 401 ]] || task13_fail "server profile did not enforce upstream auth" + authenticated="$TASK13_TMP/server-workspaces.out" + curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" --max-time "$TASK13_CURL_MAX_TIME" \ + --fail --silent --show-error \ + -H 'x-thoth-principal-issuer: task13-proxy' \ + -H 'x-thoth-principal-subject: task13-user' \ + -H 'x-thoth-principal-display-name: Task 13 User' \ + "http://$frontend/api/workspaces" >"$authenticated" + grep -Fq 'Task 13 Smoke' "$authenticated" \ + || task13_fail "authenticated server route did not expose the disposable registry" + + session_status="$(curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" \ + --max-time "$TASK13_CURL_MAX_TIME" --silent --output "$TASK13_TMP/server-sessions.out" \ + --write-out '%{http_code}' \ + -H 'x-thoth-principal-issuer: task13-proxy' \ + -H 'x-thoth-principal-subject: task13-user' \ + "http://$frontend/api/sessions")" + [[ "$session_status" == 503 ]] \ + || task13_fail "disposable unavailable session dependency did not fail closed with 503" + if grep -Fq "$TASK13_SECRET_VALUE" "$TASK13_TMP/server-sessions.out"; then + task13_fail "server session failure exposed the fixture secret" + fi +} + task13_registry_status() { task13_compose exec -T core curl --connect-timeout 3 --max-time 10 -fsS \ http://127.0.0.1:8787/workspace-registry/status @@ -523,10 +764,18 @@ task13_registry_lifecycle() { } task13_prepare_bad_candidate() { - task13_run_logged "pull pinned dead-core candidate" docker image pull "$TASK13_BAD_CANDIDATE_IMAGE" + task13_run_logged "pull pinned stopped-core candidate" docker image pull "$TASK13_BAD_CANDIDATE_IMAGE" TASK13_BAD_CANDIDATE_ID="$(docker image inspect --format '{{.Id}}' "$TASK13_BAD_CANDIDATE_IMAGE")" [[ "$TASK13_BAD_CANDIDATE_ID" =~ ^sha256:[0-9a-f]{64}$ ]] \ || task13_fail "bad candidate image identity was not resolved" + task13_run_logged "prove bad candidate exits" docker run \ + --name "$TASK13_BAD_CANDIDATE_CONTAINER" \ + --label "io.thothii.task13.run=$TASK13_RUN_ID" \ + "$TASK13_BAD_CANDIDATE_IMAGE" + [[ "$(docker container inspect --format '{{.State.Running}}:{{.State.ExitCode}}' \ + "$TASK13_BAD_CANDIDATE_CONTAINER")" == false:0 ]] \ + || task13_fail "bad candidate did not reach the guaranteed stopped state" + task13_remove_labeled_container "$TASK13_BAD_CANDIDATE_CONTAINER" } task13_update_rollback() { @@ -624,7 +873,12 @@ task13_remove_transaction_image() { task13_assert_project_ownership() { local kind id ids label for kind in container volume network; do - if ! ids="$(docker "$kind" ls -q --filter "label=com.docker.compose.project=$TASK13_PROJECT")"; then + if [[ "$kind" == container ]]; then + if ! ids="$(docker container ls -aq --filter "label=com.docker.compose.project=$TASK13_PROJECT")"; then + task13_fail "could not enumerate Compose project container resources" + return 1 + fi + elif ! ids="$(docker "$kind" ls -q --filter "label=com.docker.compose.project=$TASK13_PROJECT")"; then task13_fail "could not enumerate Compose project $kind resources" return 1 fi @@ -674,6 +928,7 @@ task13_cleanup() { printf '%s\n' '--- sanitized Task 13 diagnostic log ---' >&2 tail -n 200 "$TASK13_LOG" | task13_sanitize >&2 fi + task13_remove_labeled_container "${TASK13_BAD_CANDIDATE_CONTAINER:-}" || cleanup_rc=1 task13_remove_labeled_container "${TASK13_LLM_CONTAINER:-}" || cleanup_rc=1 if [[ -n "${TASK13_PROJECT:-}" && -n "${TASK13_ROOT:-}" && -f "${TASK13_OVERRIDE:-}" ]]; then if task13_assert_project_ownership >>"${TASK13_LOG:-/dev/null}" 2>&1; then @@ -899,6 +1154,177 @@ task13_self_test_transaction_image_cleanup() { rm -f "$calls" "$foreign_error" } +task13_self_test_rollback_fixture_contract() { + [[ "${TASK13_BAD_CANDIDATE_BEHAVIOR:-}" == stopped ]] \ + || task13_fail "rollback candidate is not declared as guaranteed stopped" + [[ "$TASK13_BAD_CANDIDATE_IMAGE" =~ ^hello-world@sha256:[0-9a-f]{64}$ ]] \ + || task13_fail "rollback candidate is not the digest-pinned stopped fixture" +} + +task13_self_test_runtime_binding_fixture() { + local fixture_source + fixture_source="$(declare -f task13_write_fixture_files)" + for variable in \ + THT_WS_TASK13_SMOKE_DWH_HOST \ + THT_WS_TASK13_SMOKE_DWH_PORT \ + THT_WS_TASK13_SMOKE_DWH_USER \ + THT_WS_TASK13_SMOKE_DWH_PASSWORD_FILE \ + THT_WS_TASK13_SMOKE_VECTOR_HOST \ + THT_WS_TASK13_SMOKE_VECTOR_PORT \ + THT_WS_TASK13_SMOKE_VECTOR_USER \ + THT_WS_TASK13_SMOKE_VECTOR_PASSWORD_FILE \ + THT_WS_TASK13_SMOKE_EMBEDDING_BASE_URL; do + grep -Fq "$variable" <<<"$fixture_source" \ + || task13_fail "rollback fixture lacks runtime binding: $variable" + done +} + +task13_self_test_server_runtime_binding_fixture() { + local fixture_source + fixture_source="$(declare -f task13_write_server_fixture_files)" + for variable in \ + THT_WS_TASK13_SMOKE_DWH_HOST \ + THT_WS_TASK13_SMOKE_DWH_PASSWORD_FILE \ + THT_WS_TASK13_SMOKE_VECTOR_HOST \ + THT_WS_TASK13_SMOKE_VECTOR_PASSWORD_FILE \ + THT_WS_TASK13_SMOKE_EMBEDDING_BASE_URL; do + grep -Fq "$variable" <<<"$fixture_source" \ + || task13_fail "server fixture lacks runtime binding: $variable" + done +} + +task13_self_test_stopped_project_containers() { + local calls foreign_error + calls="$(mktemp "${TMPDIR:-/tmp}/thothii-task13-project-containers.XXXXXX")" + foreign_error="$calls.foreign-error" + TASK13_PROJECT="thothii-0123456789ab" + TASK13_RUN_ID="task13-contract-run" + + docker() { + printf '%s\n' "$*" >>"$calls" + case "$1 $2 $3" in + "container ls -aq") printf '%s\n' stopped-foreign ;; + "container inspect --format") printf '%s\n' some-other-run ;; + "volume ls -q"|"network ls -q") : ;; + *) return 1 ;; + esac + } + + if task13_assert_project_ownership 2>"$foreign_error"; then + unset -f docker + rm -f "$calls" "$foreign_error" + task13_fail "project cleanup accepted a stopped foreign container" + fi + grep -Fq 'container ls -aq' "$calls" \ + || task13_fail "project cleanup did not enumerate stopped containers" + grep -Fq 'Compose project contains a foreign container resource' "$foreign_error" \ + || task13_fail "stopped foreign container refusal was not explicit" + + : >"$calls" + docker() { + printf '%s\n' "$*" >>"$calls" + case "$1 $2 $3" in + "container ls -aq") printf '%s\n' stopped-owned ;; + "container inspect --format") printf '%s\n' "$TASK13_RUN_ID" ;; + "volume ls -q"|"network ls -q") : ;; + *) return 1 ;; + esac + } + task13_assert_project_ownership + [[ "$(grep -Fc 'container inspect --format' "$calls")" -eq 1 ]] \ + || task13_fail "project cleanup did not inspect exactly the stopped owned container" + unset -f docker + rm -f "$calls" "$foreign_error" +} + +task13_self_test_timeout_process_group() { + local child_file child_pid="" rc + child_file="$(mktemp "${TMPDIR:-/tmp}/thothii-task13-timeout-child.XXXXXX")" + set +e + task13_bounded 1 "child-process regression" bash -c \ + 'sleep 30 & printf "%s\n" "$!" >"$1"; wait' _ "$child_file" >/dev/null 2>&1 + rc=$? + set -e + child_pid="$(sed -n '1p' "$child_file")" + [[ "$rc" -ne 0 ]] || { + rm -f "$child_file" + task13_fail "timed command unexpectedly succeeded" + } + if [[ -n "$child_pid" ]] && kill -0 "$child_pid" 2>/dev/null; then + kill -KILL "$child_pid" 2>/dev/null || true + rm -f "$child_file" + task13_fail "timed command left its child process alive" + fi + rm -f "$child_file" +} + +task13_self_test_nested_timeout_process_group() { + local child_file child_pid="" rc + child_file="$(mktemp "${TMPDIR:-/tmp}/thothii-task13-nested-timeout.XXXXXX")" + task13_nested_timeout_fixture() { + task13_bounded 30 "nested child-process regression" bash -c \ + 'sleep 30 & printf "%s\n" "$!" >"$1"; wait' _ "$child_file" + } + set +e + task13_supervise 1 "nested timeout supervisor" task13_nested_timeout_fixture >/dev/null 2>&1 + rc=$? + set -e + unset -f task13_nested_timeout_fixture + child_pid="$(sed -n '1p' "$child_file")" + [[ "$rc" -ne 0 ]] || { + rm -f "$child_file" + task13_fail "nested timed command unexpectedly succeeded" + } + if [[ -n "$child_pid" ]] && kill -0 "$child_pid" 2>/dev/null; then + kill -KILL "$child_pid" 2>/dev/null || true + rm -f "$child_file" + task13_fail "outer timeout left its nested command child alive" + fi + rm -f "$child_file" +} + +task13_self_test_public_timeout_contract() { + local root + root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)" + grep -Eq 'task13_supervise[[:space:]].*task13_smoke_main[[:space:]]+full' \ + "$root/scripts/unified-deployment-smoke.sh" \ + || task13_fail "direct unified smoke invocation lacks an internal supervisor" + grep -Eq 'task13_supervise[[:space:]].*task13_smoke_main[[:space:]]+update' \ + "$root/scripts/thothctl-update-smoke.sh" \ + || task13_fail "direct update smoke invocation lacks an internal supervisor" +} + +task13_self_test_windows_release_contract() { + local root script workflow + root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)" + script="$root/scripts/test-windows-clone-contract.ps1" + workflow="$root/.github/workflows/deployment.yml" + grep -Fq 'Task 13 path with spaces' "$script" \ + || task13_fail "Windows contract does not operate from a path containing spaces" + grep -Fq 'DockerStartup' "$script" \ + || task13_fail "Windows contract lacks an explicit Docker startup mode" + grep -Fq 'Kill($true)' "$script" \ + || task13_fail "Windows bounded runner does not kill the full process tree" + grep -Fq 'docker-desktop' "$workflow" \ + || task13_fail "workflow lacks a manual self-hosted Windows Docker Desktop gate" + grep -Fq -- '-DockerStartup' "$workflow" \ + || task13_fail "manual Windows release job does not execute Docker startup mode" +} + +task13_self_test_server_release_contract() { + local root workflow server_smoke + root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)" + workflow="$root/.github/workflows/deployment.yml" + server_smoke="$root/scripts/server-deployment-smoke.sh" + [[ -x "$server_smoke" ]] || task13_fail "bounded Linux server deployment smoke is missing" + grep -Fq 'deploy/compose.server.yaml' "$root/scripts/unified-deployment-smoke.sh" \ + || task13_fail "server smoke does not load the server profile" + grep -Fq 'deploy/compose.session-server.yaml.example' "$root/scripts/unified-deployment-smoke.sh" \ + || task13_fail "server smoke does not load the required session overlay" + grep -Eq 'timeout .*scripts/server-deployment-smoke\.sh' "$workflow" \ + || task13_fail "workflow lacks an outer timeout for the Linux server smoke" +} + task13_self_test_source_contract() { local root host_network push_command registry_function workflow uses_count pinned_uses_count root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)" @@ -950,10 +1376,34 @@ task13_self_test() { task13_self_test_cleanup_ownership task13_self_test_image_cleanup_ownership task13_self_test_transaction_image_cleanup + task13_self_test_rollback_fixture_contract + task13_self_test_runtime_binding_fixture + task13_self_test_server_runtime_binding_fixture + task13_self_test_stopped_project_containers + task13_self_test_timeout_process_group + task13_self_test_nested_timeout_process_group + task13_self_test_public_timeout_contract + task13_self_test_windows_release_contract + task13_self_test_server_release_contract task13_self_test_source_contract printf 'Task 13 smoke safety contracts passed.\n' } +task13_self_test_case() { + case "$1" in + rollback) task13_self_test_rollback_fixture_contract ;; + runtime-bindings) task13_self_test_runtime_binding_fixture ;; + server-bindings) task13_self_test_server_runtime_binding_fixture ;; + cleanup) task13_self_test_stopped_project_containers ;; + timeout-group) task13_self_test_timeout_process_group ;; + timeout-nested) task13_self_test_nested_timeout_process_group ;; + timeout-public) task13_self_test_public_timeout_contract ;; + windows) task13_self_test_windows_release_contract ;; + server) task13_self_test_server_release_contract ;; + *) task13_fail "unknown Task 13 self-test case: $1" ;; + esac +} + task13_initialize() { umask 077 TASK13_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)" @@ -966,6 +1416,7 @@ task13_initialize() { trap 'task13_cleanup $?' EXIT trap 'exit 130' INT TERM HUP TASK13_RUN_ID="$(date -u +%Y%m%d%H%M%S)-$$-${RANDOM:-0}" + TASK13_PROFILE="local" TASK13_INSTALLATION="$TASK13_TMP/thothii-installation.yaml" TASK13_PROJECT="thothii-$(task13_sha256_text "$TASK13_INSTALLATION" | cut -c1-12)" TASK13_CONTROL_DIR="$TASK13_ROOT/.thothctl/$TASK13_PROJECT" @@ -984,12 +1435,22 @@ task13_initialize() { TASK13_LLM_SERVER="$TASK13_TMP/fake-llm.mjs" TASK13_THOTHCTL_DIR="$TASK13_TMP/thothctl" TASK13_LLM_CONTAINER="$TASK13_PROJECT-llm" + TASK13_BAD_CANDIDATE_CONTAINER="$TASK13_PROJECT-bad-candidate" TASK13_CORE_IMAGE="task13-core-$TASK13_RUN_ID:local" TASK13_FRONTEND_IMAGE="task13-frontend-$TASK13_RUN_ID:local" TASK13_SECRET_VALUE="task13-secret-$TASK13_RUN_ID" TASK13_NETWORK="" TASK13_BAD_CANDIDATE_ID="" TASK13_PREVIOUS_IMAGE_ID="" + TASK13_SERVER_DATA="$TASK13_TMP/Server Data" + TASK13_SERVER_PI_STATE="$TASK13_TMP/Server Pi State" + TASK13_SERVER_REGISTRY="$TASK13_TMP/Server Registry" + TASK13_SERVER_WORKSPACE_CONFIG="$TASK13_TMP/server-sessions.yaml" + TASK13_SESSION_RUNTIME_PASSWORD="$TASK13_TMP/session-runtime-password" + TASK13_SESSION_MIGRATOR_PASSWORD_FILE="$TASK13_TMP/session-migrator-password" + TASK13_SESSION_CA="$TASK13_TMP/session-ca.pem" + TASK13_SESSION_PASSWORD="task13-runtime-$TASK13_RUN_ID" + TASK13_SESSION_MIGRATOR_PASSWORD="task13-migrator-$TASK13_RUN_ID" } task13_require_tools() { @@ -1022,10 +1483,26 @@ task13_smoke_main() { printf 'Task 13 %s deployment smoke passed.\n' "$mode" } +task13_server_smoke_main() { + task13_initialize + TASK13_PROFILE="server" + task13_require_tools + task13_write_server_fixture_files + task13_seed_registry + task13_start_server_stack + task13_assert_project_ownership + task13_assert_built_image_ownership + task13_assert_server_runtime + printf 'Task 13 Linux server deployment smoke passed.\n' +} + if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then if [[ "${1:-}" == "--self-test" ]]; then task13_self_test + elif [[ "${1:-}" == "--self-test-case" ]]; then + [[ -n "${2:-}" ]] || task13_fail "--self-test-case requires a case name" + task13_self_test_case "$2" else - task13_smoke_main full + task13_supervise "$TASK13_SMOKE_TIMEOUT" "unified deployment smoke" task13_smoke_main full fi fi diff --git a/tools/thothctl/cmd/thothctl/main.go b/tools/thothctl/cmd/thothctl/main.go index 7fb6772e..69d02447 100644 --- a/tools/thothctl/cmd/thothctl/main.go +++ b/tools/thothctl/cmd/thothctl/main.go @@ -193,6 +193,13 @@ type installationRunner struct { runner compose.Runner } +func (r installationRunner) SessionInventoryScope() string { + if r.installation.Profile == "local" { + return "mine" + } + return "all" +} + func (r installationRunner) Run(ctx context.Context, args []string, stdin io.Reader) (compose.Result, error) { if len(args) > 0 && args[0] == "compose" { return r.runner.Run(ctx, r.installation.ComposeArgs(args[1:]...), stdin) diff --git a/tools/thothctl/internal/pi/update.go b/tools/thothctl/internal/pi/update.go index 19edffb8..fd8f004d 100644 --- a/tools/thothctl/internal/pi/update.go +++ b/tools/thothctl/internal/pi/update.go @@ -526,8 +526,14 @@ func ensureMaintenance(ctx context.Context, runner Runner) error { } func activeSessions(ctx context.Context, runner Runner) (bool, error) { + scope := "all" + if scoped, ok := runner.(interface{ SessionInventoryScope() string }); ok { + if requested := scoped.SessionInventoryScope(); requested == "mine" || requested == "all" { + scope = requested + } + } args := append([]string{"exec", "-T", "core", "curl", "-fsS"}, internalIdentityHeaders...) - args = append(args, "http://127.0.0.1:8787/sessions?scope=all") + args = append(args, "http://127.0.0.1:8787/sessions?scope="+scope) result, err := runCompose(ctx, runner, args...) if err != nil { return false, commandError("active-session check", result, err) diff --git a/tools/thothctl/internal/pi/update_test.go b/tools/thothctl/internal/pi/update_test.go index 4b80651e..40a68f68 100644 --- a/tools/thothctl/internal/pi/update_test.go +++ b/tools/thothctl/internal/pi/update_test.go @@ -30,6 +30,30 @@ func TestActiveSessionsParsesAuthenticatedBackendBareArrayFixture(t *testing.T) } } +type scopedSessionRunner struct { + calls []string + scope string +} + +func (r *scopedSessionRunner) Run(_ context.Context, args []string, _ io.Reader) (compose.Result, error) { + r.calls = append(r.calls, strings.Join(args, " ")) + return compose.Result{Stdout: "[]"}, nil +} + +func (r *scopedSessionRunner) SessionInventoryScope() string { return r.scope } + +func TestActiveSessionsUsesInstallationScopedInventory(t *testing.T) { + for _, scope := range []string{"mine", "all"} { + runner := &scopedSessionRunner{scope: scope} + if active, err := activeSessions(context.Background(), runner); err != nil || active { + t.Fatalf("activeSessions(%s) = %t, %v; want false, nil", scope, active, err) + } + if len(runner.calls) != 1 || !strings.Contains(runner.calls[0], "/sessions?scope="+scope) { + t.Fatalf("activeSessions(%s) call = %v; want installation-scoped inventory", scope, runner.calls) + } + } +} + func TestUpdateBuildsPinnedVersionRecreatesOnlyCoreAndPersistsRecoveryState(t *testing.T) { fake := newFakeRunner() dir := t.TempDir()