fix: complete deployment release gates
This commit is contained in:
@@ -5,6 +5,12 @@ on:
|
|||||||
push:
|
push:
|
||||||
branches: [main]
|
branches: [main]
|
||||||
workflow_dispatch:
|
workflow_dispatch:
|
||||||
|
inputs:
|
||||||
|
windows_docker_startup:
|
||||||
|
description: Run the native self-hosted Windows Docker Desktop/WSL2 release gate
|
||||||
|
required: false
|
||||||
|
type: boolean
|
||||||
|
default: false
|
||||||
|
|
||||||
permissions:
|
permissions:
|
||||||
contents: read
|
contents: read
|
||||||
@@ -60,16 +66,18 @@ jobs:
|
|||||||
linux-docker:
|
linux-docker:
|
||||||
name: Linux Docker deployment and rollback
|
name: Linux Docker deployment and rollback
|
||||||
runs-on: ubuntu-24.04
|
runs-on: ubuntu-24.04
|
||||||
timeout-minutes: 70
|
timeout-minutes: 100
|
||||||
steps:
|
steps:
|
||||||
- name: Check out source
|
- name: Check out source
|
||||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||||
with:
|
with:
|
||||||
persist-credentials: false
|
persist-credentials: false
|
||||||
- name: Run unified deployment smoke
|
- name: Run unified deployment smoke
|
||||||
run: timeout --signal=TERM --kill-after=45s 30m bash scripts/unified-deployment-smoke.sh
|
run: timeout --signal=TERM --kill-after=45s 32m bash scripts/unified-deployment-smoke.sh
|
||||||
- name: Run thothctl update smoke
|
- name: Run thothctl update smoke
|
||||||
run: timeout --signal=TERM --kill-after=45s 30m bash scripts/thothctl-update-smoke.sh
|
run: timeout --signal=TERM --kill-after=45s 32m bash scripts/thothctl-update-smoke.sh
|
||||||
|
- name: Run Linux server deployment smoke
|
||||||
|
run: timeout --signal=TERM --kill-after=45s 32m bash scripts/server-deployment-smoke.sh
|
||||||
|
|
||||||
windows-clone:
|
windows-clone:
|
||||||
name: Windows clone and Compose contract
|
name: Windows clone and Compose contract
|
||||||
@@ -85,14 +93,25 @@ jobs:
|
|||||||
with:
|
with:
|
||||||
go-version: "1.26.5"
|
go-version: "1.26.5"
|
||||||
cache-dependency-path: tools/thothctl/go.sum
|
cache-dependency-path: tools/thothctl/go.sum
|
||||||
- name: Build native Windows thothctl
|
|
||||||
working-directory: tools/thothctl
|
|
||||||
shell: pwsh
|
|
||||||
run: |
|
|
||||||
New-Item -ItemType Directory -Force -Path ../../dist/thothctl | Out-Null
|
|
||||||
go build -trimpath -o ../../dist/thothctl/thothctl-windows-amd64.exe ./cmd/thothctl
|
|
||||||
- name: Verify Windows clone contract
|
- name: Verify Windows clone contract
|
||||||
shell: pwsh
|
shell: pwsh
|
||||||
run: >-
|
run: ./scripts/test-windows-clone-contract.ps1
|
||||||
./scripts/test-windows-clone-contract.ps1
|
|
||||||
-ThothctlPath "$PWD/dist/thothctl/thothctl-windows-amd64.exe"
|
windows-docker-release:
|
||||||
|
name: Native Windows Docker Desktop/WSL2 startup
|
||||||
|
if: github.event_name == 'workflow_dispatch' && inputs.windows_docker_startup
|
||||||
|
runs-on: [self-hosted, Windows, X64, docker-desktop]
|
||||||
|
timeout-minutes: 45
|
||||||
|
steps:
|
||||||
|
- name: Check out source
|
||||||
|
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||||
|
with:
|
||||||
|
persist-credentials: false
|
||||||
|
- name: Set up Go
|
||||||
|
uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
|
||||||
|
with:
|
||||||
|
go-version: "1.26.5"
|
||||||
|
cache-dependency-path: tools/thothctl/go.sum
|
||||||
|
- name: Run spaced-path Windows Docker release gate
|
||||||
|
shell: pwsh
|
||||||
|
run: ./scripts/test-windows-clone-contract.ps1 -DockerStartup
|
||||||
|
|||||||
+25
-16
@@ -9,28 +9,37 @@
|
|||||||
frontend-to-core routing, embedded pinned Pi, Git registry bootstrap, offline recreation, valid
|
frontend-to-core routing, embedded pinned Pi, Git registry bootstrap, offline recreation, valid
|
||||||
update, invalid-update retention, and the four persistent stores. `scripts/thothctl-update-smoke.sh`
|
update, invalid-update retention, and the four persistent stores. `scripts/thothctl-update-smoke.sh`
|
||||||
independently exercises the bad-Pi update and automatic rollback path.
|
independently exercises the bad-Pi update and automatic rollback path.
|
||||||
|
`scripts/server-deployment-smoke.sh` starts the server plus required session overlays with the
|
||||||
|
same smoke-built core/frontend images, disposable bind roots/secrets/session configuration,
|
||||||
|
upstream-auth checks, and fail-closed unavailable-session behavior.
|
||||||
- **Isolation and disclosure boundary.** Every run generates a unique temporary root, Compose
|
- **Isolation and disclosure boundary.** Every run generates a unique temporary root, Compose
|
||||||
project, container/image names, transaction image tags, and run label. The rollback fixture uses
|
project, container/image names, transaction image tags, and run label. The rollback fixture uses
|
||||||
an immutable public digest as a deliberately dead core rather than a host-local image registry.
|
an immutable `hello-world` digest whose preflight exits successfully, guaranteeing the stopped
|
||||||
Cleanup checks ownership before removing exact containers, Compose resources, image references,
|
core state required by `thothctl` compensation. Cleanup includes stopped project containers in
|
||||||
control state, and temporary files. There is no global prune. Failure diagnostics are bounded
|
its final ownership check immediately before teardown and removes only exact containers,
|
||||||
and sanitized, and all credentials/endpoints used by the smokes are disposable fixtures rather
|
Compose resources, image references, control state, and temporary files. There is no global
|
||||||
than operator or repository secrets.
|
prune. Failure diagnostics are bounded and sanitized, and all credentials/endpoints used by the
|
||||||
|
smokes are disposable fixtures rather than operator or repository secrets. Every public smoke
|
||||||
|
also has an internal 30-minute process-group supervisor with TERM/KILL of the complete group.
|
||||||
- **Cross-platform CI contract.** `.github/workflows/deployment.yml` uses immutable action commits,
|
- **Cross-platform CI contract.** `.github/workflows/deployment.yml` uses immutable action commits,
|
||||||
pinned supported Node and Go versions, runs LF/Compose/secret/coupling/docs/TypeScript gates on
|
pinned supported Node and Go versions, runs LF/Compose/secret/coupling/docs/TypeScript gates on
|
||||||
Linux, runs each Docker smoke once under its own outer timeout, and builds/invokes native Windows
|
Linux, runs each Linux Docker smoke once under its own outer timeout, and copies the Windows
|
||||||
`thothctl` after the PowerShell clone/LF/Compose contract. Native Windows execution remains an
|
source into a path containing spaces before building/invoking native `thothctl` and rendering
|
||||||
explicit manual release gate in addition to CI; no Windows Docker container startup is claimed
|
Compose. The optional `windows_docker_startup` dispatch targets a labelled self-hosted Windows
|
||||||
by the static clone job.
|
Docker Desktop/WSL2 runner and performs bounded two-service startup and exact cleanup. No local
|
||||||
|
Windows or Windows Docker execution is claimed until that manual job is recorded.
|
||||||
- **Validation status.** Deterministic Phase A gates, backend **434/434** plus TypeScript,
|
- **Validation status.** Deterministic Phase A gates, backend **434/434** plus TypeScript,
|
||||||
frontend **386/386** plus TypeScript, and harness **862 passed / 5 L2 deselected** are green.
|
frontend **386/386** plus TypeScript, and harness **862 passed / 5 L2 deselected** are green.
|
||||||
The unified one-shot Docker run passed frontend/core/internal Pi, registry bootstrap, offline
|
Review round 1 ran each Docker smoke exactly once without retry. Unified (`103.86s`) and
|
||||||
recreation, valid update, invalid-update retention, and persistence before Docker Desktop
|
update-only (`46.45s`) passed build/start, core/Pi/registry/persistence setup and the stopped
|
||||||
refused the daemon-to-host local-registry push; the update-only run reached the same boundary.
|
candidate preflight, but `thothctl` stopped before mutation at its active-session inventory gate.
|
||||||
Both exact run/project resource sets were independently proved absent. The local-registry
|
A test-first fix now scopes local inventory to `mine` and supplies the fixture's missing direct
|
||||||
fixture was then removed in favor of the immutable dead-core digest, but the requested no-retry
|
DWH/vector/embedding runtime bindings; the final rollback path was not rerun, so compensation
|
||||||
rule leaves automatic rollback/preservation pending in CI or a fresh manual release run. Native
|
and all-sentinel preservation remain unproven. Server-profile execution (`12.88s`) built both
|
||||||
Windows PowerShell execution is also still a manual release gate.
|
images but Docker Desktop/VirtioFS rejected the real profile's parent Pi-state bind plus nested
|
||||||
|
tracked agent-file binds before service startup. Every run's exact labelled cleanup passed.
|
||||||
|
Native-Linux server startup and native Windows PowerShell/Docker execution remain explicit
|
||||||
|
CI/manual release gates; no local success is claimed for either platform.
|
||||||
|
|
||||||
## Portable deployment decoupling — LIVE 2026-08-05
|
## Portable deployment decoupling — LIVE 2026-08-05
|
||||||
|
|
||||||
|
|||||||
@@ -91,42 +91,67 @@ later with `docker compose --project-name "$SMOKE_PROJECT" down --volumes`.
|
|||||||
|
|
||||||
## Unified deployment release gates
|
## Unified deployment release gates
|
||||||
|
|
||||||
Task 13 adds a no-secret release gate around the canonical base plus local Compose profile. Its
|
Task 13 adds no-secret release gates around the canonical local and server Compose profiles. Its
|
||||||
deterministic safety check does not contact the Docker daemon:
|
deterministic safety check does not contact the Docker daemon:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
bash scripts/unified-deployment-smoke.sh --self-test
|
bash scripts/unified-deployment-smoke.sh --self-test
|
||||||
```
|
```
|
||||||
|
|
||||||
The two Docker smokes are separate release jobs. Each creates a unique Compose project, temporary
|
The three Linux Docker smokes are separate release commands. Each creates a unique Compose project, temporary
|
||||||
Git workspace remote, fixture provider, image names, and run label. Its exit trap removes only
|
Git workspace remote, fixture provider, image names, and run label. Its exit trap removes only
|
||||||
resources carrying that exact run identity and never performs a global Docker prune.
|
resources carrying that exact run identity and never performs a global Docker prune. Cleanup
|
||||||
|
enumerates running and stopped project containers immediately before `compose down` and refuses
|
||||||
|
the teardown if any container, volume, or network has a foreign run label.
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
bash scripts/unified-deployment-smoke.sh
|
bash scripts/unified-deployment-smoke.sh
|
||||||
bash scripts/thothctl-update-smoke.sh
|
bash scripts/thothctl-update-smoke.sh
|
||||||
|
bash scripts/server-deployment-smoke.sh
|
||||||
```
|
```
|
||||||
|
|
||||||
The unified smoke builds and starts `frontend` and `core`, verifies the embedded Pi and internal
|
The unified smoke builds and starts `frontend` and `core`, verifies the embedded Pi and internal
|
||||||
registry, recreates with the Git remote offline, activates a valid Git update, rejects invalid Git
|
registry, recreates with the Git remote offline, activates a valid Git update, rejects invalid Git
|
||||||
content while retaining the valid snapshot, and checks the four persistence volumes. Both smokes
|
content while retaining the valid snapshot, and checks the four persistence volumes. The unified
|
||||||
|
and update-only smokes
|
||||||
inject a digest-pinned non-core candidate under a deliberately mismatched Pi version and require
|
inject a digest-pinned non-core candidate under a deliberately mismatched Pi version and require
|
||||||
`thothctl pi update` to roll back while preserving settings, sessions, Pi state, registry revision,
|
`thothctl pi update` to roll back while preserving settings, sessions, Pi state, registry revision,
|
||||||
and mount identity. Fixture credentials are generated locally; neither command needs a real
|
and mount identity. The rollback candidate is the digest-pinned `hello-world` executable: a
|
||||||
provider key or a repository secret. CI gives each smoke one 30-minute outer timeout and does not
|
preflight proves that it exits successfully, so the failed replacement core satisfies
|
||||||
retry it.
|
`thothctl`'s stopped-core compensation precondition. The server smoke uses the same smoke-built
|
||||||
|
core/frontend images with the server and required session overlays, disposable bind roots and
|
||||||
|
secret files, upstream-auth checks, and a fail-closed `503` assertion for its deliberately
|
||||||
|
unavailable disposable session endpoint. No real provider, database credential, or repository
|
||||||
|
secret is required.
|
||||||
|
|
||||||
On a native Windows clone, the release contract is:
|
Each public smoke has its own 30-minute process-group supervisor with TERM/KILL cleanup; CI retains
|
||||||
|
an independent 32-minute outer timeout and does not retry a failed command.
|
||||||
|
|
||||||
|
Current release status (2026-08-05): deterministic contracts are green, but the complete rollback
|
||||||
|
fixture has not passed end to end after its runtime-binding correction. The one observed local
|
||||||
|
server-profile run also stopped before startup because Docker Desktop/VirtioFS rejected the
|
||||||
|
profile's parent Pi-state bind with nested tracked agent-file binds. A fresh single rollback run,
|
||||||
|
native-Linux server-profile run, and native Windows Docker Desktop/WSL2 run remain release gates;
|
||||||
|
the project does not claim those criteria green.
|
||||||
|
|
||||||
|
The deterministic native Windows contract is:
|
||||||
|
|
||||||
```powershell
|
```powershell
|
||||||
.\scripts\test-windows-clone-contract.ps1 `
|
.\scripts\test-windows-clone-contract.ps1
|
||||||
-ThothctlPath "$PWD\dist\thothctl\thothctl-windows-amd64.exe"
|
|
||||||
```
|
```
|
||||||
|
|
||||||
It checks Git's CRLF/LF attributes and bytes, renders exactly `core` plus `frontend` with Docker
|
It checks Git's CRLF/LF attributes and bytes, copies tracked source into a temporary path containing
|
||||||
Compose without starting containers, and invokes the native Windows `thothctl`. The GitHub Actions
|
spaces, builds and invokes native Windows `thothctl` there, and renders exactly `core` plus
|
||||||
deployment workflow runs the deterministic Linux gates, both bounded Docker smokes, and this
|
`frontend` without starting containers. On a supported self-hosted Windows Docker Desktop/WSL2
|
||||||
Windows clone contract with immutable action pins and supported pinned Node/Go toolchains.
|
runner, dispatch the deployment workflow with `windows_docker_startup=true`; that job executes:
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
.\scripts\test-windows-clone-contract.ps1 -DockerStartup
|
||||||
|
```
|
||||||
|
|
||||||
|
Startup mode adds bounded image build/two-service health startup, installation-aware `thothctl`
|
||||||
|
status, stopped-container-aware ownership checks, and exact cleanup. The ordinary hosted Windows
|
||||||
|
job remains deterministic and does not claim Docker startup.
|
||||||
|
|
||||||
## Optional local pgvector and recovery
|
## Optional local pgvector and recovery
|
||||||
|
|
||||||
|
|||||||
Executable
+8
@@ -0,0 +1,8 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
root="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||||
|
# shellcheck source=./unified-deployment-smoke.sh
|
||||||
|
source "$root/scripts/unified-deployment-smoke.sh"
|
||||||
|
|
||||||
|
task13_supervise "$TASK13_SMOKE_TIMEOUT" "Linux server deployment smoke" task13_server_smoke_main
|
||||||
@@ -1,27 +1,90 @@
|
|||||||
param(
|
param(
|
||||||
[string]$RepositoryRoot = "",
|
[string]$RepositoryRoot = "",
|
||||||
[string]$ThothctlPath = ""
|
[switch]$DockerStartup,
|
||||||
|
[int]$CommandTimeoutSeconds = 600
|
||||||
)
|
)
|
||||||
|
|
||||||
$ErrorActionPreference = "Stop"
|
$ErrorActionPreference = "Stop"
|
||||||
Set-StrictMode -Version Latest
|
Set-StrictMode -Version Latest
|
||||||
|
$script:SensitiveValues = [System.Collections.Generic.List[string]]::new()
|
||||||
|
|
||||||
|
function Protect-Output([string]$Value) {
|
||||||
|
$protected = $Value
|
||||||
|
foreach ($secret in $script:SensitiveValues) {
|
||||||
|
if (-not [string]::IsNullOrEmpty($secret)) {
|
||||||
|
$protected = $protected.Replace($secret, "[REDACTED]")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return $protected -replace '(?i)((?:password|token|api[_-]?key|secret|key)\s*[:=]\s*)[^\s,;]+', '$1[REDACTED]'
|
||||||
|
}
|
||||||
|
|
||||||
|
function Invoke-BoundedNative {
|
||||||
|
param(
|
||||||
|
[Parameter(Mandatory = $true)][string]$FilePath,
|
||||||
|
[Parameter(Mandatory = $true)][string[]]$Arguments,
|
||||||
|
[Parameter(Mandatory = $true)][string]$Label,
|
||||||
|
[string]$WorkingDirectory = "",
|
||||||
|
[int]$TimeoutSeconds = $CommandTimeoutSeconds,
|
||||||
|
[switch]$AllowFailure
|
||||||
|
)
|
||||||
|
$startInfo = [System.Diagnostics.ProcessStartInfo]::new()
|
||||||
|
$startInfo.FileName = $FilePath
|
||||||
|
$startInfo.UseShellExecute = $false
|
||||||
|
$startInfo.RedirectStandardOutput = $true
|
||||||
|
$startInfo.RedirectStandardError = $true
|
||||||
|
$startInfo.CreateNoWindow = $true
|
||||||
|
if (-not [string]::IsNullOrWhiteSpace($WorkingDirectory)) {
|
||||||
|
$startInfo.WorkingDirectory = $WorkingDirectory
|
||||||
|
}
|
||||||
|
foreach ($argument in $Arguments) {
|
||||||
|
[void]$startInfo.ArgumentList.Add($argument)
|
||||||
|
}
|
||||||
|
$process = [System.Diagnostics.Process]::new()
|
||||||
|
$process.StartInfo = $startInfo
|
||||||
|
if (-not $process.Start()) {
|
||||||
|
throw "$Label could not start"
|
||||||
|
}
|
||||||
|
$stdoutTask = $process.StandardOutput.ReadToEndAsync()
|
||||||
|
$stderrTask = $process.StandardError.ReadToEndAsync()
|
||||||
|
if (-not $process.WaitForExit($TimeoutSeconds * 1000)) {
|
||||||
|
$process.Kill($true)
|
||||||
|
[void]$process.WaitForExit(30000)
|
||||||
|
throw "$Label timed out after $TimeoutSeconds seconds"
|
||||||
|
}
|
||||||
|
$stdout = $stdoutTask.GetAwaiter().GetResult()
|
||||||
|
$stderr = $stderrTask.GetAwaiter().GetResult()
|
||||||
|
$result = [pscustomobject]@{
|
||||||
|
ExitCode = $process.ExitCode
|
||||||
|
StdOut = $stdout
|
||||||
|
StdErr = $stderr
|
||||||
|
}
|
||||||
|
if (-not $AllowFailure -and $result.ExitCode -ne 0) {
|
||||||
|
$diagnostic = Protect-Output (($result.StdOut + "`n" + $result.StdErr).Trim())
|
||||||
|
throw "$Label failed with exit $($result.ExitCode): $diagnostic"
|
||||||
|
}
|
||||||
|
return $result
|
||||||
|
}
|
||||||
|
|
||||||
|
function Write-Utf8File([string]$Path, [string]$Contents) {
|
||||||
|
[System.IO.Directory]::CreateDirectory([System.IO.Path]::GetDirectoryName($Path)) | Out-Null
|
||||||
|
[System.IO.File]::WriteAllText($Path, $Contents, [System.Text.UTF8Encoding]::new($false))
|
||||||
|
}
|
||||||
|
|
||||||
|
function ConvertTo-YamlPath([string]$Path) {
|
||||||
|
return $Path.Replace('\', '/').Replace('"', '\"')
|
||||||
|
}
|
||||||
|
|
||||||
if ([string]::IsNullOrWhiteSpace($RepositoryRoot)) {
|
if ([string]::IsNullOrWhiteSpace($RepositoryRoot)) {
|
||||||
$RepositoryRoot = (& git rev-parse --show-toplevel).Trim()
|
$resolved = Invoke-BoundedNative -FilePath "git" -Arguments @("rev-parse", "--show-toplevel") -Label "resolve repository root"
|
||||||
if ($LASTEXITCODE -ne 0) {
|
$RepositoryRoot = $resolved.StdOut.Trim()
|
||||||
throw "git could not resolve the repository root"
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
$RepositoryRoot = [System.IO.Path]::GetFullPath($RepositoryRoot)
|
$RepositoryRoot = [System.IO.Path]::GetFullPath($RepositoryRoot)
|
||||||
|
|
||||||
$tracked = @(& git -C $RepositoryRoot ls-files)
|
$trackedResult = Invoke-BoundedNative -FilePath "git" -Arguments @("-C", $RepositoryRoot, "ls-files") -Label "list tracked files"
|
||||||
if ($LASTEXITCODE -ne 0) {
|
$tracked = @($trackedResult.StdOut -split "`r?`n" | Where-Object { -not [string]::IsNullOrWhiteSpace($_) })
|
||||||
throw "git ls-files failed"
|
|
||||||
}
|
|
||||||
|
|
||||||
$scriptRelativePath = "scripts/test-windows-clone-contract.ps1"
|
$scriptRelativePath = "scripts/test-windows-clone-contract.ps1"
|
||||||
$eolAttribute = (& git -C $RepositoryRoot check-attr eol -- $scriptRelativePath).Trim()
|
$attribute = Invoke-BoundedNative -FilePath "git" -Arguments @("-C", $RepositoryRoot, "check-attr", "eol", "--", $scriptRelativePath) -Label "read PowerShell eol attribute"
|
||||||
if ($LASTEXITCODE -ne 0 -or -not $eolAttribute.EndsWith("eol: crlf", [System.StringComparison]::OrdinalIgnoreCase)) {
|
if (-not $attribute.StdOut.Trim().EndsWith("eol: crlf", [System.StringComparison]::OrdinalIgnoreCase)) {
|
||||||
throw "the Windows contract script must have the repository eol=crlf attribute"
|
throw "the Windows contract script must have the repository eol=crlf attribute"
|
||||||
}
|
}
|
||||||
$scriptBytes = [System.IO.File]::ReadAllBytes((Join-Path $RepositoryRoot $scriptRelativePath))
|
$scriptBytes = [System.IO.File]::ReadAllBytes((Join-Path $RepositoryRoot $scriptRelativePath))
|
||||||
@@ -38,71 +101,203 @@ foreach ($relativePath in $tracked) {
|
|||||||
$name.Equals("Dockerfile", [System.StringComparison]::OrdinalIgnoreCase) -or
|
$name.Equals("Dockerfile", [System.StringComparison]::OrdinalIgnoreCase) -or
|
||||||
$name.StartsWith("Dockerfile.", [System.StringComparison]::OrdinalIgnoreCase) -or
|
$name.StartsWith("Dockerfile.", [System.StringComparison]::OrdinalIgnoreCase) -or
|
||||||
$name.EndsWith(".Dockerfile", [System.StringComparison]::OrdinalIgnoreCase)
|
$name.EndsWith(".Dockerfile", [System.StringComparison]::OrdinalIgnoreCase)
|
||||||
if (-not $mustBeLf) {
|
if ($mustBeLf) {
|
||||||
continue
|
$bytes = [System.IO.File]::ReadAllBytes((Join-Path $RepositoryRoot $relativePath))
|
||||||
}
|
|
||||||
$absolutePath = Join-Path $RepositoryRoot $relativePath
|
|
||||||
$bytes = [System.IO.File]::ReadAllBytes($absolutePath)
|
|
||||||
if ($bytes -contains [byte]0x0D) {
|
if ($bytes -contains [byte]0x0D) {
|
||||||
$offenders.Add($relativePath)
|
$offenders.Add($relativePath)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
if ($offenders.Count -ne 0) {
|
if ($offenders.Count -ne 0) {
|
||||||
throw "CR byte 0x0D found in tracked LF contract files: $($offenders -join ', ')"
|
throw "CR byte 0x0D found in tracked LF contract files: $($offenders -join ', ')"
|
||||||
}
|
}
|
||||||
|
|
||||||
$temporaryRoot = Join-Path ([System.IO.Path]::GetTempPath()) ("thothii-windows-contract-" + [guid]::NewGuid().ToString("N"))
|
$runId = [guid]::NewGuid().ToString("N")
|
||||||
$savedEnvironment = @{
|
$temporaryRoot = Join-Path ([System.IO.Path]::GetTempPath()) ("ThothII Task 13 path with spaces " + $runId)
|
||||||
THT_WORKSPACE_GIT_REMOTE = $env:THT_WORKSPACE_GIT_REMOTE
|
$spacedRepository = Join-Path $temporaryRoot "Task 13 path with spaces"
|
||||||
PI_AUTH_FILE = $env:PI_AUTH_FILE
|
$fixtureRoot = Join-Path $temporaryRoot "Disposable Fixture Data"
|
||||||
THT_SECRETS_FILE = $env:THT_SECRETS_FILE
|
$project = "thothii-win-" + $runId.Substring(0, 12)
|
||||||
}
|
$runLabel = "windows-" + $runId
|
||||||
try {
|
$coreImage = "task13-windows-core-$($runId.Substring(0, 16)):local"
|
||||||
[System.IO.Directory]::CreateDirectory($temporaryRoot) | Out-Null
|
$frontendImage = "task13-windows-frontend-$($runId.Substring(0, 16)):local"
|
||||||
$piAuth = Join-Path $temporaryRoot "pi-auth.json"
|
$composeArguments = @()
|
||||||
$secrets = Join-Path $temporaryRoot "thothii.secrets"
|
$startupAttempted = $false
|
||||||
[System.IO.File]::WriteAllText($piAuth, "{}`n", [System.Text.UTF8Encoding]::new($false))
|
$cleanupSucceeded = $true
|
||||||
[System.IO.File]::WriteAllText($secrets, "THT_MODEL_API_KEY=windows-contract`n", [System.Text.UTF8Encoding]::new($false))
|
$savedEnvironment = @{}
|
||||||
|
|
||||||
$env:THT_WORKSPACE_GIT_REMOTE = "https://git.example.invalid/platform/thoth-workspaces.git"
|
try {
|
||||||
$env:PI_AUTH_FILE = $piAuth
|
[System.IO.Directory]::CreateDirectory($spacedRepository) | Out-Null
|
||||||
$env:THT_SECRETS_FILE = $secrets
|
foreach ($relativePath in $tracked) {
|
||||||
$composeFiles = @(
|
$source = Join-Path $RepositoryRoot $relativePath
|
||||||
"--project-directory", $RepositoryRoot,
|
$destination = Join-Path $spacedRepository $relativePath
|
||||||
"-f", (Join-Path $RepositoryRoot "compose.yaml"),
|
[System.IO.Directory]::CreateDirectory([System.IO.Path]::GetDirectoryName($destination)) | Out-Null
|
||||||
"-f", (Join-Path $RepositoryRoot "deploy/compose.local.yaml")
|
Copy-Item -LiteralPath $source -Destination $destination
|
||||||
)
|
|
||||||
$services = @(& docker compose @composeFiles config --services)
|
|
||||||
if ($LASTEXITCODE -ne 0) {
|
|
||||||
throw "Docker Compose could not render the Windows clone"
|
|
||||||
}
|
}
|
||||||
if ((($services | Sort-Object) -join ",") -ne "core,frontend") {
|
|
||||||
|
$thothctl = Join-Path $spacedRepository "dist/thothctl/thothctl-windows-amd64.exe"
|
||||||
|
[System.IO.Directory]::CreateDirectory([System.IO.Path]::GetDirectoryName($thothctl)) | Out-Null
|
||||||
|
Invoke-BoundedNative -FilePath "go" -Arguments @("build", "-trimpath", "-o", $thothctl, "./cmd/thothctl") `
|
||||||
|
-WorkingDirectory (Join-Path $spacedRepository "tools/thothctl") -Label "build native Windows thothctl in spaced path" | Out-Null
|
||||||
|
Invoke-BoundedNative -FilePath $thothctl -Arguments @("--help") -Label "invoke native Windows thothctl from spaced path" | Out-Null
|
||||||
|
|
||||||
|
$piAuth = Join-Path $fixtureRoot "Pi Auth/pi-auth.json"
|
||||||
|
$secrets = Join-Path $fixtureRoot "Secrets/thothii.secrets"
|
||||||
|
$secretValue = "windows-contract-$runId"
|
||||||
|
$script:SensitiveValues.Add($secretValue)
|
||||||
|
Write-Utf8File $piAuth "{}`n"
|
||||||
|
Write-Utf8File $secrets "THT_MODEL_API_KEY=$secretValue`n"
|
||||||
|
|
||||||
|
$remote = Join-Path $fixtureRoot "Workspace Remote/remote.git"
|
||||||
|
$seed = Join-Path $fixtureRoot "Workspace Seed"
|
||||||
|
[System.IO.Directory]::CreateDirectory((Join-Path $seed "workspaces")) | Out-Null
|
||||||
|
Invoke-BoundedNative -FilePath "git" -Arguments @("init", "--bare", "--initial-branch=main", $remote) -Label "initialize Windows bare registry" | Out-Null
|
||||||
|
Invoke-BoundedNative -FilePath "git" -Arguments @("-C", $seed, "init", "--initial-branch=main") -Label "initialize Windows registry seed" | Out-Null
|
||||||
|
Write-Utf8File (Join-Path $seed "workspaces/task13-windows.yaml") @"
|
||||||
|
workspace:
|
||||||
|
schema_version: 2
|
||||||
|
id: task13-windows
|
||||||
|
name: Task 13 Windows
|
||||||
|
language: en
|
||||||
|
dwh:
|
||||||
|
engine: postgres
|
||||||
|
database: warehouse
|
||||||
|
schema: public
|
||||||
|
supported_transports: [postgres_direct]
|
||||||
|
"@
|
||||||
|
Invoke-BoundedNative -FilePath "git" -Arguments @("-C", $seed, "add", "workspaces/task13-windows.yaml") -Label "stage Windows registry seed" | Out-Null
|
||||||
|
Invoke-BoundedNative -FilePath "git" -Arguments @("-C", $seed, "-c", "user.name=Task 13 Windows", "-c", "user.email=task13-windows@example.invalid", "commit", "-m", "Seed Windows smoke") -Label "commit Windows registry seed" | Out-Null
|
||||||
|
Invoke-BoundedNative -FilePath "git" -Arguments @("-C", $seed, "push", $remote, "HEAD:main") -Label "push Windows registry seed" | Out-Null
|
||||||
|
|
||||||
|
$envFile = Join-Path $fixtureRoot "Config/local.env"
|
||||||
|
$override = Join-Path $fixtureRoot "Config/compose.windows.yaml"
|
||||||
|
$installation = Join-Path $fixtureRoot "Config/thothii installation.yaml"
|
||||||
|
Write-Utf8File $envFile @"
|
||||||
|
THOTH_HTTP_PORT=0
|
||||||
|
THOTH_CORE_HTTP_PORT=0
|
||||||
|
PI_AUTH_FILE=$piAuth
|
||||||
|
THT_SECRETS_FILE=$secrets
|
||||||
|
THT_WORKSPACE_GIT_REMOTE=/fixtures/remote.git
|
||||||
|
THT_WORKSPACE_GIT_BRANCH=main
|
||||||
|
THT_WORKSPACE_INSTALLATION_ID=task13-windows
|
||||||
|
"@
|
||||||
|
$remoteYaml = ConvertTo-YamlPath $remote
|
||||||
|
Write-Utf8File $override @"
|
||||||
|
services:
|
||||||
|
core:
|
||||||
|
image: $coreImage
|
||||||
|
build:
|
||||||
|
labels:
|
||||||
|
io.thothii.task13.run: "$runLabel"
|
||||||
|
labels:
|
||||||
|
io.thothii.task13.run: "$runLabel"
|
||||||
|
volumes:
|
||||||
|
- "$remoteYaml:/fixtures/remote.git:ro"
|
||||||
|
frontend:
|
||||||
|
image: $frontendImage
|
||||||
|
build:
|
||||||
|
labels:
|
||||||
|
io.thothii.task13.run: "$runLabel"
|
||||||
|
labels:
|
||||||
|
io.thothii.task13.run: "$runLabel"
|
||||||
|
networks:
|
||||||
|
thothii:
|
||||||
|
labels:
|
||||||
|
io.thothii.task13.run: "$runLabel"
|
||||||
|
volumes:
|
||||||
|
settings:
|
||||||
|
labels:
|
||||||
|
io.thothii.task13.run: "$runLabel"
|
||||||
|
pi-state:
|
||||||
|
labels:
|
||||||
|
io.thothii.task13.run: "$runLabel"
|
||||||
|
workspace-registry:
|
||||||
|
labels:
|
||||||
|
io.thothii.task13.run: "$runLabel"
|
||||||
|
sessions:
|
||||||
|
labels:
|
||||||
|
io.thothii.task13.run: "$runLabel"
|
||||||
|
"@
|
||||||
|
$repoYaml = ConvertTo-YamlPath $spacedRepository
|
||||||
|
$envYaml = ConvertTo-YamlPath $envFile
|
||||||
|
$overrideYaml = ConvertTo-YamlPath $override
|
||||||
|
Write-Utf8File $installation @"
|
||||||
|
profile: local
|
||||||
|
projectDirectory: "$repoYaml"
|
||||||
|
envFile: "$envYaml"
|
||||||
|
overrides:
|
||||||
|
- "$overrideYaml"
|
||||||
|
"@
|
||||||
|
|
||||||
|
$composeArguments = @(
|
||||||
|
"compose", "--project-name", $project, "--project-directory", $spacedRepository,
|
||||||
|
"--env-file", $envFile,
|
||||||
|
"-f", (Join-Path $spacedRepository "compose.yaml"),
|
||||||
|
"-f", (Join-Path $spacedRepository "deploy/compose.local.yaml"),
|
||||||
|
"-f", $override
|
||||||
|
)
|
||||||
|
$render = Invoke-BoundedNative -FilePath "docker" -Arguments ($composeArguments + @("config", "--services")) -Label "render Windows Compose from spaced path"
|
||||||
|
$services = @($render.StdOut -split "`r?`n" | Where-Object { $_ } | Sort-Object)
|
||||||
|
if (($services -join ",") -ne "core,frontend") {
|
||||||
throw "rendered Windows stack must contain exactly core and frontend"
|
throw "rendered Windows stack must contain exactly core and frontend"
|
||||||
}
|
}
|
||||||
& docker compose @composeFiles config --quiet
|
Invoke-BoundedNative -FilePath "docker" -Arguments ($composeArguments + @("config", "--quiet")) -Label "validate Windows Compose from spaced path" | Out-Null
|
||||||
if ($LASTEXITCODE -ne 0) {
|
|
||||||
throw "Docker Compose rejected the Windows clone"
|
|
||||||
}
|
|
||||||
|
|
||||||
if ([string]::IsNullOrWhiteSpace($ThothctlPath)) {
|
if ($DockerStartup) {
|
||||||
$ThothctlPath = Join-Path $RepositoryRoot "dist/thothctl/thothctl-windows-amd64.exe"
|
Invoke-BoundedNative -FilePath "docker" -Arguments @("info") -Label "verify Windows Docker Desktop readiness" -TimeoutSeconds 60 | Out-Null
|
||||||
|
$startupAttempted = $true
|
||||||
|
Invoke-BoundedNative -FilePath "docker" -Arguments ($composeArguments + @("build", "--pull")) -Label "build two-service Windows stack" | Out-Null
|
||||||
|
Invoke-BoundedNative -FilePath "docker" -Arguments ($composeArguments + @("up", "--detach", "--wait", "--wait-timeout", "180")) -Label "start two-service Windows stack" -TimeoutSeconds 300 | Out-Null
|
||||||
|
$running = Invoke-BoundedNative -FilePath "docker" -Arguments ($composeArguments + @("ps", "--status", "running", "--services")) -Label "inspect running Windows services"
|
||||||
|
$runningServices = @($running.StdOut -split "`r?`n" | Where-Object { $_ } | Sort-Object)
|
||||||
|
if (($runningServices -join ",") -ne "core,frontend") {
|
||||||
|
throw "bounded Windows startup did not leave exactly core and frontend running"
|
||||||
}
|
}
|
||||||
$ThothctlPath = [System.IO.Path]::GetFullPath($ThothctlPath)
|
Invoke-BoundedNative -FilePath $thothctl -Arguments @("--installation", $installation, "status") -Label "invoke installation-aware Windows thothctl in spaced path" | Out-Null
|
||||||
if (-not [System.IO.File]::Exists($ThothctlPath)) {
|
|
||||||
throw "Windows thothctl binary is missing: $ThothctlPath"
|
|
||||||
}
|
|
||||||
& $ThothctlPath --help | Out-Null
|
|
||||||
if ($LASTEXITCODE -ne 0) {
|
|
||||||
throw "Windows thothctl invocation failed"
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
finally {
|
finally {
|
||||||
|
if ($startupAttempted -and $composeArguments.Count -ne 0) {
|
||||||
|
try {
|
||||||
|
foreach ($kind in @("container", "volume", "network")) {
|
||||||
|
$listArgs = if ($kind -eq "container") { @($kind, "ls", "-aq") } else { @($kind, "ls", "-q") }
|
||||||
|
$listed = Invoke-BoundedNative -FilePath "docker" -Arguments ($listArgs + @("--filter", "label=com.docker.compose.project=$project")) -Label "enumerate Windows project $kind resources" -TimeoutSeconds 30
|
||||||
|
foreach ($id in @($listed.StdOut -split "`r?`n" | Where-Object { $_ })) {
|
||||||
|
$format = if ($kind -eq "container") { '{{ index .Config.Labels "io.thothii.task13.run" }}' } else { '{{ index .Labels "io.thothii.task13.run" }}' }
|
||||||
|
$inspected = Invoke-BoundedNative -FilePath "docker" -Arguments @($kind, "inspect", "--format", $format, $id) -Label "inspect Windows project $kind ownership" -TimeoutSeconds 30
|
||||||
|
if ($inspected.StdOut.Trim() -ne $runLabel) {
|
||||||
|
throw "refusing to remove foreign Windows project $kind resource"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Invoke-BoundedNative -FilePath "docker" -Arguments ($composeArguments + @("down", "--volumes", "--remove-orphans", "--timeout", "10")) -Label "remove exact Windows Compose project" -TimeoutSeconds 60 | Out-Null
|
||||||
|
foreach ($image in @($frontendImage, $coreImage)) {
|
||||||
|
$inspection = Invoke-BoundedNative -FilePath "docker" -Arguments @("image", "inspect", "--format", '{{ index .Config.Labels "io.thothii.task13.run" }}', $image) -Label "inspect Windows image ownership" -TimeoutSeconds 30 -AllowFailure
|
||||||
|
if ($inspection.ExitCode -eq 0) {
|
||||||
|
if ($inspection.StdOut.Trim() -ne $runLabel) {
|
||||||
|
throw "refusing to remove foreign Windows image $image"
|
||||||
|
}
|
||||||
|
Invoke-BoundedNative -FilePath "docker" -Arguments @("image", "rm", $image) -Label "remove exact Windows image" -TimeoutSeconds 60 | Out-Null
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch {
|
||||||
|
$cleanupSucceeded = $false
|
||||||
|
Write-Error (Protect-Output $_.Exception.Message)
|
||||||
|
}
|
||||||
|
}
|
||||||
foreach ($name in $savedEnvironment.Keys) {
|
foreach ($name in $savedEnvironment.Keys) {
|
||||||
[System.Environment]::SetEnvironmentVariable($name, $savedEnvironment[$name], "Process")
|
[System.Environment]::SetEnvironmentVariable($name, $savedEnvironment[$name], "Process")
|
||||||
}
|
}
|
||||||
if ([System.IO.Directory]::Exists($temporaryRoot)) {
|
if ($cleanupSucceeded -and [System.IO.Directory]::Exists($temporaryRoot)) {
|
||||||
Remove-Item -LiteralPath $temporaryRoot -Recurse -Force
|
Remove-Item -LiteralPath $temporaryRoot -Recurse -Force
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
Write-Output "Windows clone, LF-byte, Compose render, and thothctl invocation contracts passed."
|
if (-not $cleanupSucceeded) {
|
||||||
|
throw "Windows cleanup proof failed; fixture path retained for recovery"
|
||||||
|
}
|
||||||
|
if ($DockerStartup) {
|
||||||
|
Write-Output "Windows spaced-path build, native thothctl, bounded two-service startup, and exact cleanup passed."
|
||||||
|
} else {
|
||||||
|
Write-Output "Windows spaced-path clone, LF-byte, Compose render, and native thothctl build/invocation contracts passed; Docker startup mode was not requested."
|
||||||
|
}
|
||||||
|
|||||||
@@ -5,4 +5,4 @@ root="$(cd "$(dirname "$0")/.." && pwd -P)"
|
|||||||
# shellcheck source=./unified-deployment-smoke.sh
|
# shellcheck source=./unified-deployment-smoke.sh
|
||||||
source "$root/scripts/unified-deployment-smoke.sh"
|
source "$root/scripts/unified-deployment-smoke.sh"
|
||||||
|
|
||||||
task13_smoke_main update
|
task13_supervise "$TASK13_SMOKE_TIMEOUT" "thothctl update smoke" task13_smoke_main update
|
||||||
|
|||||||
@@ -4,11 +4,15 @@
|
|||||||
# isolated fixture, exact cleanup, and sanitized failure reporting.
|
# isolated fixture, exact cleanup, and sanitized failure reporting.
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
|
|
||||||
TASK13_BAD_CANDIDATE_IMAGE="registry:2.8.3@sha256:a3d8aaa63ed8681a604f1dea0aa03f100d5895b6a58ace528858a7b332415373"
|
TASK13_BAD_CANDIDATE_IMAGE="hello-world@sha256:c3cbe1cc1aa588a64951ac6286e0df7b27fe2e6324b1001c619bb358770c0178"
|
||||||
|
TASK13_BAD_CANDIDATE_BEHAVIOR="stopped"
|
||||||
TASK13_BAD_PI_VERSION="0.80.4-task13"
|
TASK13_BAD_PI_VERSION="0.80.4-task13"
|
||||||
TASK13_CURL_CONNECT_TIMEOUT=3
|
TASK13_CURL_CONNECT_TIMEOUT=3
|
||||||
TASK13_CURL_MAX_TIME=10
|
TASK13_CURL_MAX_TIME=10
|
||||||
TASK13_CLEANUP_TIMEOUT=20
|
TASK13_CLEANUP_TIMEOUT=20
|
||||||
|
TASK13_COMMAND_TIMEOUT=900
|
||||||
|
TASK13_SMOKE_TIMEOUT=1800
|
||||||
|
TASK13_TERM_GRACE=45
|
||||||
|
|
||||||
task13_fail() {
|
task13_fail() {
|
||||||
printf 'Task 13 smoke failed: %s\n' "$*" >&2
|
printf 'Task 13 smoke failed: %s\n' "$*" >&2
|
||||||
@@ -49,41 +53,82 @@ task13_log_failure() {
|
|||||||
task13_run_logged() {
|
task13_run_logged() {
|
||||||
local label="$1"
|
local label="$1"
|
||||||
shift
|
shift
|
||||||
if ! "$@" >>"$TASK13_LOG" 2>&1; then
|
if ! task13_bounded "$TASK13_COMMAND_TIMEOUT" "$label" "$@" >>"$TASK13_LOG" 2>&1; then
|
||||||
task13_log_failure "$label"
|
task13_log_failure "$label"
|
||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
task13_bounded() {
|
task13_bounded() {
|
||||||
local seconds="$1" label="$2" command_pid watchdog_pid rc
|
local seconds="$1" label="$2" command_pid watchdog_pid rc watchdog_rc=0
|
||||||
|
local monitor_enabled=0 timeout_marker command_group
|
||||||
shift 2
|
shift 2
|
||||||
|
timeout_marker="$(mktemp "${TMPDIR:-/tmp}/thothii-task13-timeout.XXXXXX")"
|
||||||
|
[[ $- == *m* ]] && monitor_enabled=1
|
||||||
|
if [[ -n "${TASK13_ACTIVE_GROUP:-}" ]]; then
|
||||||
|
[[ "$TASK13_ACTIVE_GROUP" =~ ^[0-9]+$ ]] \
|
||||||
|
|| task13_fail "invalid active Task 13 process group"
|
||||||
|
set +m
|
||||||
"$@" &
|
"$@" &
|
||||||
command_pid=$!
|
command_pid=$!
|
||||||
|
command_group="$TASK13_ACTIVE_GROUP"
|
||||||
|
else
|
||||||
|
set -m
|
||||||
|
"$@" &
|
||||||
|
command_pid=$!
|
||||||
|
command_group="$command_pid"
|
||||||
|
[[ "$monitor_enabled" -eq 1 ]] || set +m
|
||||||
|
fi
|
||||||
(
|
(
|
||||||
local sleep_pid
|
local sleep_pid grace_deadline
|
||||||
sleep "$seconds" &
|
sleep "$seconds" &
|
||||||
sleep_pid=$!
|
sleep_pid=$!
|
||||||
trap 'kill "$sleep_pid" 2>/dev/null || true' EXIT INT TERM
|
trap 'kill "$sleep_pid" 2>/dev/null || true' EXIT INT TERM
|
||||||
wait "$sleep_pid" 2>/dev/null || exit 0
|
wait "$sleep_pid" 2>/dev/null || exit 0
|
||||||
trap - EXIT INT TERM
|
trap - EXIT INT TERM
|
||||||
if kill -0 "$command_pid" 2>/dev/null; then
|
if kill -0 -- "-$command_group" 2>/dev/null; then
|
||||||
|
trap '' TERM
|
||||||
|
printf 'timeout\n' >"$timeout_marker"
|
||||||
printf 'Task 13 command timed out after %ss: %s\n' "$seconds" "$label" >&2
|
printf 'Task 13 command timed out after %ss: %s\n' "$seconds" "$label" >&2
|
||||||
kill -TERM "$command_pid" 2>/dev/null || true
|
kill -TERM -- "-$command_group" 2>/dev/null || true
|
||||||
sleep 5
|
grace_deadline=$((SECONDS + TASK13_TERM_GRACE))
|
||||||
kill -KILL "$command_pid" 2>/dev/null || true
|
while kill -0 -- "-$command_group" 2>/dev/null && ((SECONDS < grace_deadline)); do
|
||||||
|
sleep 1
|
||||||
|
done
|
||||||
|
kill -KILL -- "-$command_group" 2>/dev/null || true
|
||||||
|
exit 124
|
||||||
fi
|
fi
|
||||||
) &
|
) &
|
||||||
watchdog_pid=$!
|
watchdog_pid=$!
|
||||||
|
if [[ -n "${TASK13_ACTIVE_GROUP:-}" && "$monitor_enabled" -eq 1 ]]; then
|
||||||
|
set -m
|
||||||
|
fi
|
||||||
if wait "$command_pid"; then
|
if wait "$command_pid"; then
|
||||||
rc=0
|
rc=0
|
||||||
else
|
else
|
||||||
rc=$?
|
rc=$?
|
||||||
fi
|
fi
|
||||||
|
if [[ -s "$timeout_marker" ]]; then
|
||||||
|
wait "$watchdog_pid" 2>/dev/null || watchdog_rc=$?
|
||||||
|
else
|
||||||
kill "$watchdog_pid" 2>/dev/null || true
|
kill "$watchdog_pid" 2>/dev/null || true
|
||||||
wait "$watchdog_pid" 2>/dev/null || true
|
wait "$watchdog_pid" 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
rm -f "$timeout_marker"
|
||||||
|
[[ "$watchdog_rc" -eq 124 ]] && return 124
|
||||||
return "$rc"
|
return "$rc"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
task13_supervised_call() {
|
||||||
|
TASK13_ACTIVE_GROUP="$BASHPID"
|
||||||
|
"$@"
|
||||||
|
}
|
||||||
|
|
||||||
|
task13_supervise() {
|
||||||
|
local seconds="$1" label="$2"
|
||||||
|
shift 2
|
||||||
|
task13_bounded "$seconds" "$label" task13_supervised_call "$@"
|
||||||
|
}
|
||||||
|
|
||||||
task13_compose_files() {
|
task13_compose_files() {
|
||||||
TASK13_COMPOSE=(
|
TASK13_COMPOSE=(
|
||||||
docker compose
|
docker compose
|
||||||
@@ -91,10 +136,20 @@ task13_compose_files() {
|
|||||||
--project-directory "$TASK13_ROOT"
|
--project-directory "$TASK13_ROOT"
|
||||||
--env-file "$TASK13_ENV_FILE"
|
--env-file "$TASK13_ENV_FILE"
|
||||||
-f "$TASK13_ROOT/compose.yaml"
|
-f "$TASK13_ROOT/compose.yaml"
|
||||||
|
)
|
||||||
|
if [[ "${TASK13_PROFILE:-local}" == server ]]; then
|
||||||
|
TASK13_COMPOSE+=(
|
||||||
|
-f "$TASK13_ROOT/deploy/compose.server.yaml"
|
||||||
|
-f "$TASK13_ROOT/deploy/compose.session-server.yaml.example"
|
||||||
|
-f "$TASK13_OVERRIDE"
|
||||||
|
)
|
||||||
|
else
|
||||||
|
TASK13_COMPOSE+=(
|
||||||
-f "$TASK13_ROOT/deploy/compose.local.yaml"
|
-f "$TASK13_ROOT/deploy/compose.local.yaml"
|
||||||
-f "$TASK13_OVERRIDE"
|
-f "$TASK13_OVERRIDE"
|
||||||
)
|
)
|
||||||
if [[ -f "$TASK13_CURRENT_IMAGE_OVERRIDE" ]]; then
|
fi
|
||||||
|
if [[ "${TASK13_PROFILE:-local}" == local && -f "$TASK13_CURRENT_IMAGE_OVERRIDE" ]]; then
|
||||||
TASK13_COMPOSE+=(-f "$TASK13_CURRENT_IMAGE_OVERRIDE")
|
TASK13_COMPOSE+=(-f "$TASK13_CURRENT_IMAGE_OVERRIDE")
|
||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
@@ -224,6 +279,15 @@ services:
|
|||||||
PI_THINKING: low
|
PI_THINKING: low
|
||||||
THT_WORKSPACE_INSTALLATION_ID: task13-smoke
|
THT_WORKSPACE_INSTALLATION_ID: task13-smoke
|
||||||
THT_WORKSPACE_REGISTRY_ROOT: /data/workspace-registry
|
THT_WORKSPACE_REGISTRY_ROOT: /data/workspace-registry
|
||||||
|
THT_WS_TASK13_SMOKE_DWH_HOST: dwh.task13.invalid
|
||||||
|
THT_WS_TASK13_SMOKE_DWH_PORT: "5432"
|
||||||
|
THT_WS_TASK13_SMOKE_DWH_USER: task13_reader
|
||||||
|
THT_WS_TASK13_SMOKE_DWH_PASSWORD_FILE: /run/secrets/thothii.secrets
|
||||||
|
THT_WS_TASK13_SMOKE_VECTOR_HOST: vector.task13.invalid
|
||||||
|
THT_WS_TASK13_SMOKE_VECTOR_PORT: "5432"
|
||||||
|
THT_WS_TASK13_SMOKE_VECTOR_USER: task13_vector_reader
|
||||||
|
THT_WS_TASK13_SMOKE_VECTOR_PASSWORD_FILE: /run/secrets/thothii.secrets
|
||||||
|
THT_WS_TASK13_SMOKE_EMBEDDING_BASE_URL: http://$TASK13_LLM_CONTAINER:9000
|
||||||
labels:
|
labels:
|
||||||
io.thothii.task13.run: "$TASK13_RUN_ID"
|
io.thothii.task13.run: "$TASK13_RUN_ID"
|
||||||
volumes: !override
|
volumes: !override
|
||||||
@@ -272,6 +336,114 @@ EOF
|
|||||||
chmod 0600 "$TASK13_INSTALLATION"
|
chmod 0600 "$TASK13_INSTALLATION"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
task13_write_server_fixture_files() {
|
||||||
|
local data_root pi_root registry_root remote_path workspace_path
|
||||||
|
printf '{}\n' >"$TASK13_PI_AUTH"
|
||||||
|
printf 'THT_MODEL_API_KEY=%s\n' "$TASK13_SECRET_VALUE" >"$TASK13_SECRETS"
|
||||||
|
printf '%s\n' "$TASK13_SESSION_PASSWORD" >"$TASK13_SESSION_RUNTIME_PASSWORD"
|
||||||
|
printf '%s\n' "$TASK13_SESSION_MIGRATOR_PASSWORD" >"$TASK13_SESSION_MIGRATOR_PASSWORD_FILE"
|
||||||
|
cat >"$TASK13_SESSION_CA" <<'EOF'
|
||||||
|
-----BEGIN CERTIFICATE-----
|
||||||
|
VEFTSzEzLURJU1BPU0FCTEUtU0VTU0lPTi1DQQ==
|
||||||
|
-----END CERTIFICATE-----
|
||||||
|
EOF
|
||||||
|
chmod 0644 "$TASK13_PI_AUTH" "$TASK13_SESSION_CA"
|
||||||
|
chmod 0600 "$TASK13_SECRETS" "$TASK13_SESSION_RUNTIME_PASSWORD" \
|
||||||
|
"$TASK13_SESSION_MIGRATOR_PASSWORD_FILE"
|
||||||
|
|
||||||
|
cat >"$TASK13_SERVER_WORKSPACE_CONFIG" <<'EOF'
|
||||||
|
language: en
|
||||||
|
session_storage:
|
||||||
|
type: postgres_direct
|
||||||
|
connection:
|
||||||
|
host: ${THT_SESSION_DB_HOST}
|
||||||
|
port: ${THT_SESSION_DB_PORT}
|
||||||
|
database: ${THT_SESSION_DB_NAME}
|
||||||
|
schema: thoth_sessions
|
||||||
|
user: ${THT_SESSION_RUNTIME_USER}
|
||||||
|
password_file: ${THT_SESSION_RUNTIME_PASSWORD_FILE}
|
||||||
|
sslmode: ${THT_SESSION_DB_SSLMODE}
|
||||||
|
sslrootcert: ${THT_SESSION_DB_SSLROOTCERT}
|
||||||
|
roots:
|
||||||
|
artifacts: artifacts
|
||||||
|
indexes: indexes
|
||||||
|
sessions: sessions
|
||||||
|
EOF
|
||||||
|
chmod 0600 "$TASK13_SERVER_WORKSPACE_CONFIG"
|
||||||
|
|
||||||
|
mkdir -p "$TASK13_SERVER_DATA" "$TASK13_SERVER_PI_STATE" "$TASK13_SERVER_REGISTRY"
|
||||||
|
chmod 0777 "$TASK13_SERVER_DATA" "$TASK13_SERVER_PI_STATE" "$TASK13_SERVER_REGISTRY"
|
||||||
|
data_root="$TASK13_SERVER_DATA"
|
||||||
|
pi_root="$TASK13_SERVER_PI_STATE"
|
||||||
|
registry_root="$TASK13_SERVER_REGISTRY"
|
||||||
|
remote_path="$TASK13_REMOTE"
|
||||||
|
workspace_path="$TASK13_SERVER_WORKSPACE_CONFIG"
|
||||||
|
|
||||||
|
cat >"$TASK13_OVERRIDE" <<EOF
|
||||||
|
services:
|
||||||
|
core:
|
||||||
|
image: $TASK13_CORE_IMAGE
|
||||||
|
build:
|
||||||
|
labels:
|
||||||
|
io.thothii.task13.run: "$TASK13_RUN_ID"
|
||||||
|
environment:
|
||||||
|
THT_WS_TASK13_SMOKE_DWH_HOST: dwh.task13.invalid
|
||||||
|
THT_WS_TASK13_SMOKE_DWH_PORT: "5432"
|
||||||
|
THT_WS_TASK13_SMOKE_DWH_USER: task13_reader
|
||||||
|
THT_WS_TASK13_SMOKE_DWH_PASSWORD_FILE: /run/secrets/thothii.secrets
|
||||||
|
THT_WS_TASK13_SMOKE_VECTOR_HOST: vector.task13.invalid
|
||||||
|
THT_WS_TASK13_SMOKE_VECTOR_PORT: "5432"
|
||||||
|
THT_WS_TASK13_SMOKE_VECTOR_USER: task13_vector_reader
|
||||||
|
THT_WS_TASK13_SMOKE_VECTOR_PASSWORD_FILE: /run/secrets/thothii.secrets
|
||||||
|
THT_WS_TASK13_SMOKE_EMBEDDING_BASE_URL: https://embedding.task13.invalid
|
||||||
|
labels:
|
||||||
|
io.thothii.task13.run: "$TASK13_RUN_ID"
|
||||||
|
volumes:
|
||||||
|
- $remote_path:/fixtures/remote.git:ro
|
||||||
|
frontend:
|
||||||
|
image: $TASK13_FRONTEND_IMAGE
|
||||||
|
build:
|
||||||
|
labels:
|
||||||
|
io.thothii.task13.run: "$TASK13_RUN_ID"
|
||||||
|
labels:
|
||||||
|
io.thothii.task13.run: "$TASK13_RUN_ID"
|
||||||
|
session-migrate:
|
||||||
|
image: $TASK13_CORE_IMAGE
|
||||||
|
networks:
|
||||||
|
thothii:
|
||||||
|
labels:
|
||||||
|
io.thothii.task13.run: "$TASK13_RUN_ID"
|
||||||
|
EOF
|
||||||
|
chmod 0600 "$TASK13_OVERRIDE"
|
||||||
|
|
||||||
|
{
|
||||||
|
printf 'THOTH_HTTP_PORT=0\n'
|
||||||
|
printf 'THOTH_SERVER_BIND=127.0.0.1\n'
|
||||||
|
printf 'MAX_PI_PROCESSES=2\n'
|
||||||
|
printf 'PI_AUTH_FILE=%s\n' "$TASK13_PI_AUTH"
|
||||||
|
printf 'THT_SECRETS_FILE=%s\n' "$TASK13_SECRETS"
|
||||||
|
printf 'THT_WORKSPACE_GIT_REMOTE=/fixtures/remote.git\n'
|
||||||
|
printf 'THT_WORKSPACE_GIT_BRANCH=%s\n' "$TASK13_BRANCH"
|
||||||
|
printf 'THT_WORKSPACE_GIT_AUTHOR_NAME=Task 13 Server Smoke\n'
|
||||||
|
printf 'THT_WORKSPACE_GIT_AUTHOR_EMAIL=task13-server@example.invalid\n'
|
||||||
|
printf 'THT_DATA_ROOT=%s\n' "$data_root"
|
||||||
|
printf 'THT_PI_STATE_ROOT=%s\n' "$pi_root"
|
||||||
|
printf 'THT_WORKSPACE_REGISTRY_ROOT=%s\n' "$registry_root"
|
||||||
|
printf 'THT_SERVER_WORKSPACE_CONFIG=%s\n' "$workspace_path"
|
||||||
|
printf 'THT_SESSION_DB_HOST=task13-session.invalid\n'
|
||||||
|
printf 'THT_SESSION_DB_PORT=5432\n'
|
||||||
|
printf 'THT_SESSION_DB_NAME=task13\n'
|
||||||
|
printf 'THT_SESSION_RUNTIME_USER=task13_runtime\n'
|
||||||
|
printf 'THT_SESSION_MIGRATOR_USER=task13_migrator\n'
|
||||||
|
printf 'THT_SESSION_DB_SSLMODE=verify-full\n'
|
||||||
|
printf 'THT_SESSION_RUNTIME_PASSWORD_SOURCE=%s\n' "$TASK13_SESSION_RUNTIME_PASSWORD"
|
||||||
|
printf 'THT_SESSION_MIGRATOR_PASSWORD_SOURCE=%s\n' "$TASK13_SESSION_MIGRATOR_PASSWORD_FILE"
|
||||||
|
printf 'THT_SESSION_CA_SOURCE=%s\n' "$TASK13_SESSION_CA"
|
||||||
|
printf 'THT_LLM_URL=https://llm.task13.invalid/v1\n'
|
||||||
|
} >"$TASK13_ENV_FILE"
|
||||||
|
chmod 0600 "$TASK13_ENV_FILE"
|
||||||
|
}
|
||||||
|
|
||||||
task13_seed_registry() {
|
task13_seed_registry() {
|
||||||
mkdir -p "$TASK13_SEED/workspaces"
|
mkdir -p "$TASK13_SEED/workspaces"
|
||||||
task13_run_logged "initialize bare workspace registry" \
|
task13_run_logged "initialize bare workspace registry" \
|
||||||
@@ -377,6 +549,14 @@ task13_start_stack() {
|
|||||||
task13_log_failure "deterministic local LLM fixture readiness"
|
task13_log_failure "deterministic local LLM fixture readiness"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
task13_start_server_stack() {
|
||||||
|
printf '== Build and start isolated Linux server profile ==\n'
|
||||||
|
task13_assert_rendered_contract
|
||||||
|
task13_compose_logged "build server Compose images" build --pull core frontend
|
||||||
|
task13_compose_logged "start server Compose distribution" \
|
||||||
|
up --detach --wait --wait-timeout 120 core frontend
|
||||||
|
}
|
||||||
|
|
||||||
task13_frontend_address() {
|
task13_frontend_address() {
|
||||||
task13_compose port frontend 8080 | awk 'NR == 1 {print $0}'
|
task13_compose port frontend 8080 | awk 'NR == 1 {print $0}'
|
||||||
}
|
}
|
||||||
@@ -422,6 +602,67 @@ task13_assert_runtime() {
|
|||||||
task13_run_logged "thothctl Pi doctor" "$TASK13_THOTHCTL" --installation "$TASK13_INSTALLATION" pi doctor
|
task13_run_logged "thothctl Pi doctor" "$TASK13_THOTHCTL" --installation "$TASK13_INSTALLATION" pi doctor
|
||||||
}
|
}
|
||||||
|
|
||||||
|
task13_assert_server_runtime() {
|
||||||
|
local frontend unauthenticated authenticated session_status core_id frontend_id
|
||||||
|
local expected_core_image expected_frontend_image
|
||||||
|
frontend="$(task13_frontend_address)"
|
||||||
|
task13_run_logged "server frontend health" curl \
|
||||||
|
--connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" --max-time "$TASK13_CURL_MAX_TIME" \
|
||||||
|
--fail --silent --show-error "http://$frontend/"
|
||||||
|
task13_run_logged "server same-origin core health" curl \
|
||||||
|
--connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" --max-time "$TASK13_CURL_MAX_TIME" \
|
||||||
|
--fail --silent --show-error "http://$frontend/api/health"
|
||||||
|
|
||||||
|
core_id="$(task13_core_id)"
|
||||||
|
frontend_id="$(task13_compose ps -q frontend)"
|
||||||
|
expected_core_image="$(docker image inspect --format '{{.Id}}' "$TASK13_CORE_IMAGE")"
|
||||||
|
expected_frontend_image="$(docker image inspect --format '{{.Id}}' "$TASK13_FRONTEND_IMAGE")"
|
||||||
|
[[ "$(docker inspect --format '{{.Image}}' "$core_id")" == "$expected_core_image" ]] \
|
||||||
|
|| task13_fail "server core did not use the smoke-built core image"
|
||||||
|
[[ "$(docker inspect --format '{{.Image}}' "$frontend_id")" == "$expected_frontend_image" ]] \
|
||||||
|
|| task13_fail "server frontend did not use the smoke-built frontend image"
|
||||||
|
task13_compose exec -T core sh -ceu '
|
||||||
|
test "$AUTH_MODE" = upstream
|
||||||
|
test "$THT_SESSION_STORAGE" = postgres
|
||||||
|
test -r /run/secrets/thothii.secrets
|
||||||
|
test -r /run/secrets/session_runtime_password
|
||||||
|
test -r /run/secrets/session_ca.pem
|
||||||
|
test -r /app/harness/workspaces/server-sessions.yaml
|
||||||
|
'
|
||||||
|
task13_mount_fingerprint | grep -Fq '/data = bind :' \
|
||||||
|
|| task13_fail "server profile did not bind the disposable data root"
|
||||||
|
task13_mount_fingerprint | grep -Fq '/home/thoth/.pi = bind :' \
|
||||||
|
|| task13_fail "server profile did not bind the disposable Pi state root"
|
||||||
|
task13_mount_fingerprint | grep -Fq '/data/workspace-registry = bind :' \
|
||||||
|
|| task13_fail "server profile did not bind the disposable registry root"
|
||||||
|
|
||||||
|
unauthenticated="$(curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" \
|
||||||
|
--max-time "$TASK13_CURL_MAX_TIME" --silent --output /dev/null --write-out '%{http_code}' \
|
||||||
|
"http://$frontend/api/workspaces")"
|
||||||
|
[[ "$unauthenticated" == 401 ]] || task13_fail "server profile did not enforce upstream auth"
|
||||||
|
authenticated="$TASK13_TMP/server-workspaces.out"
|
||||||
|
curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" --max-time "$TASK13_CURL_MAX_TIME" \
|
||||||
|
--fail --silent --show-error \
|
||||||
|
-H 'x-thoth-principal-issuer: task13-proxy' \
|
||||||
|
-H 'x-thoth-principal-subject: task13-user' \
|
||||||
|
-H 'x-thoth-principal-display-name: Task 13 User' \
|
||||||
|
"http://$frontend/api/workspaces" >"$authenticated"
|
||||||
|
grep -Fq 'Task 13 Smoke' "$authenticated" \
|
||||||
|
|| task13_fail "authenticated server route did not expose the disposable registry"
|
||||||
|
|
||||||
|
session_status="$(curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" \
|
||||||
|
--max-time "$TASK13_CURL_MAX_TIME" --silent --output "$TASK13_TMP/server-sessions.out" \
|
||||||
|
--write-out '%{http_code}' \
|
||||||
|
-H 'x-thoth-principal-issuer: task13-proxy' \
|
||||||
|
-H 'x-thoth-principal-subject: task13-user' \
|
||||||
|
"http://$frontend/api/sessions")"
|
||||||
|
[[ "$session_status" == 503 ]] \
|
||||||
|
|| task13_fail "disposable unavailable session dependency did not fail closed with 503"
|
||||||
|
if grep -Fq "$TASK13_SECRET_VALUE" "$TASK13_TMP/server-sessions.out"; then
|
||||||
|
task13_fail "server session failure exposed the fixture secret"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
task13_registry_status() {
|
task13_registry_status() {
|
||||||
task13_compose exec -T core curl --connect-timeout 3 --max-time 10 -fsS \
|
task13_compose exec -T core curl --connect-timeout 3 --max-time 10 -fsS \
|
||||||
http://127.0.0.1:8787/workspace-registry/status
|
http://127.0.0.1:8787/workspace-registry/status
|
||||||
@@ -523,10 +764,18 @@ task13_registry_lifecycle() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
task13_prepare_bad_candidate() {
|
task13_prepare_bad_candidate() {
|
||||||
task13_run_logged "pull pinned dead-core candidate" docker image pull "$TASK13_BAD_CANDIDATE_IMAGE"
|
task13_run_logged "pull pinned stopped-core candidate" docker image pull "$TASK13_BAD_CANDIDATE_IMAGE"
|
||||||
TASK13_BAD_CANDIDATE_ID="$(docker image inspect --format '{{.Id}}' "$TASK13_BAD_CANDIDATE_IMAGE")"
|
TASK13_BAD_CANDIDATE_ID="$(docker image inspect --format '{{.Id}}' "$TASK13_BAD_CANDIDATE_IMAGE")"
|
||||||
[[ "$TASK13_BAD_CANDIDATE_ID" =~ ^sha256:[0-9a-f]{64}$ ]] \
|
[[ "$TASK13_BAD_CANDIDATE_ID" =~ ^sha256:[0-9a-f]{64}$ ]] \
|
||||||
|| task13_fail "bad candidate image identity was not resolved"
|
|| task13_fail "bad candidate image identity was not resolved"
|
||||||
|
task13_run_logged "prove bad candidate exits" docker run \
|
||||||
|
--name "$TASK13_BAD_CANDIDATE_CONTAINER" \
|
||||||
|
--label "io.thothii.task13.run=$TASK13_RUN_ID" \
|
||||||
|
"$TASK13_BAD_CANDIDATE_IMAGE"
|
||||||
|
[[ "$(docker container inspect --format '{{.State.Running}}:{{.State.ExitCode}}' \
|
||||||
|
"$TASK13_BAD_CANDIDATE_CONTAINER")" == false:0 ]] \
|
||||||
|
|| task13_fail "bad candidate did not reach the guaranteed stopped state"
|
||||||
|
task13_remove_labeled_container "$TASK13_BAD_CANDIDATE_CONTAINER"
|
||||||
}
|
}
|
||||||
|
|
||||||
task13_update_rollback() {
|
task13_update_rollback() {
|
||||||
@@ -624,7 +873,12 @@ task13_remove_transaction_image() {
|
|||||||
task13_assert_project_ownership() {
|
task13_assert_project_ownership() {
|
||||||
local kind id ids label
|
local kind id ids label
|
||||||
for kind in container volume network; do
|
for kind in container volume network; do
|
||||||
if ! ids="$(docker "$kind" ls -q --filter "label=com.docker.compose.project=$TASK13_PROJECT")"; then
|
if [[ "$kind" == container ]]; then
|
||||||
|
if ! ids="$(docker container ls -aq --filter "label=com.docker.compose.project=$TASK13_PROJECT")"; then
|
||||||
|
task13_fail "could not enumerate Compose project container resources"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
elif ! ids="$(docker "$kind" ls -q --filter "label=com.docker.compose.project=$TASK13_PROJECT")"; then
|
||||||
task13_fail "could not enumerate Compose project $kind resources"
|
task13_fail "could not enumerate Compose project $kind resources"
|
||||||
return 1
|
return 1
|
||||||
fi
|
fi
|
||||||
@@ -674,6 +928,7 @@ task13_cleanup() {
|
|||||||
printf '%s\n' '--- sanitized Task 13 diagnostic log ---' >&2
|
printf '%s\n' '--- sanitized Task 13 diagnostic log ---' >&2
|
||||||
tail -n 200 "$TASK13_LOG" | task13_sanitize >&2
|
tail -n 200 "$TASK13_LOG" | task13_sanitize >&2
|
||||||
fi
|
fi
|
||||||
|
task13_remove_labeled_container "${TASK13_BAD_CANDIDATE_CONTAINER:-}" || cleanup_rc=1
|
||||||
task13_remove_labeled_container "${TASK13_LLM_CONTAINER:-}" || cleanup_rc=1
|
task13_remove_labeled_container "${TASK13_LLM_CONTAINER:-}" || cleanup_rc=1
|
||||||
if [[ -n "${TASK13_PROJECT:-}" && -n "${TASK13_ROOT:-}" && -f "${TASK13_OVERRIDE:-}" ]]; then
|
if [[ -n "${TASK13_PROJECT:-}" && -n "${TASK13_ROOT:-}" && -f "${TASK13_OVERRIDE:-}" ]]; then
|
||||||
if task13_assert_project_ownership >>"${TASK13_LOG:-/dev/null}" 2>&1; then
|
if task13_assert_project_ownership >>"${TASK13_LOG:-/dev/null}" 2>&1; then
|
||||||
@@ -899,6 +1154,177 @@ task13_self_test_transaction_image_cleanup() {
|
|||||||
rm -f "$calls" "$foreign_error"
|
rm -f "$calls" "$foreign_error"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
task13_self_test_rollback_fixture_contract() {
|
||||||
|
[[ "${TASK13_BAD_CANDIDATE_BEHAVIOR:-}" == stopped ]] \
|
||||||
|
|| task13_fail "rollback candidate is not declared as guaranteed stopped"
|
||||||
|
[[ "$TASK13_BAD_CANDIDATE_IMAGE" =~ ^hello-world@sha256:[0-9a-f]{64}$ ]] \
|
||||||
|
|| task13_fail "rollback candidate is not the digest-pinned stopped fixture"
|
||||||
|
}
|
||||||
|
|
||||||
|
task13_self_test_runtime_binding_fixture() {
|
||||||
|
local fixture_source
|
||||||
|
fixture_source="$(declare -f task13_write_fixture_files)"
|
||||||
|
for variable in \
|
||||||
|
THT_WS_TASK13_SMOKE_DWH_HOST \
|
||||||
|
THT_WS_TASK13_SMOKE_DWH_PORT \
|
||||||
|
THT_WS_TASK13_SMOKE_DWH_USER \
|
||||||
|
THT_WS_TASK13_SMOKE_DWH_PASSWORD_FILE \
|
||||||
|
THT_WS_TASK13_SMOKE_VECTOR_HOST \
|
||||||
|
THT_WS_TASK13_SMOKE_VECTOR_PORT \
|
||||||
|
THT_WS_TASK13_SMOKE_VECTOR_USER \
|
||||||
|
THT_WS_TASK13_SMOKE_VECTOR_PASSWORD_FILE \
|
||||||
|
THT_WS_TASK13_SMOKE_EMBEDDING_BASE_URL; do
|
||||||
|
grep -Fq "$variable" <<<"$fixture_source" \
|
||||||
|
|| task13_fail "rollback fixture lacks runtime binding: $variable"
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
task13_self_test_server_runtime_binding_fixture() {
|
||||||
|
local fixture_source
|
||||||
|
fixture_source="$(declare -f task13_write_server_fixture_files)"
|
||||||
|
for variable in \
|
||||||
|
THT_WS_TASK13_SMOKE_DWH_HOST \
|
||||||
|
THT_WS_TASK13_SMOKE_DWH_PASSWORD_FILE \
|
||||||
|
THT_WS_TASK13_SMOKE_VECTOR_HOST \
|
||||||
|
THT_WS_TASK13_SMOKE_VECTOR_PASSWORD_FILE \
|
||||||
|
THT_WS_TASK13_SMOKE_EMBEDDING_BASE_URL; do
|
||||||
|
grep -Fq "$variable" <<<"$fixture_source" \
|
||||||
|
|| task13_fail "server fixture lacks runtime binding: $variable"
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
task13_self_test_stopped_project_containers() {
|
||||||
|
local calls foreign_error
|
||||||
|
calls="$(mktemp "${TMPDIR:-/tmp}/thothii-task13-project-containers.XXXXXX")"
|
||||||
|
foreign_error="$calls.foreign-error"
|
||||||
|
TASK13_PROJECT="thothii-0123456789ab"
|
||||||
|
TASK13_RUN_ID="task13-contract-run"
|
||||||
|
|
||||||
|
docker() {
|
||||||
|
printf '%s\n' "$*" >>"$calls"
|
||||||
|
case "$1 $2 $3" in
|
||||||
|
"container ls -aq") printf '%s\n' stopped-foreign ;;
|
||||||
|
"container inspect --format") printf '%s\n' some-other-run ;;
|
||||||
|
"volume ls -q"|"network ls -q") : ;;
|
||||||
|
*) return 1 ;;
|
||||||
|
esac
|
||||||
|
}
|
||||||
|
|
||||||
|
if task13_assert_project_ownership 2>"$foreign_error"; then
|
||||||
|
unset -f docker
|
||||||
|
rm -f "$calls" "$foreign_error"
|
||||||
|
task13_fail "project cleanup accepted a stopped foreign container"
|
||||||
|
fi
|
||||||
|
grep -Fq 'container ls -aq' "$calls" \
|
||||||
|
|| task13_fail "project cleanup did not enumerate stopped containers"
|
||||||
|
grep -Fq 'Compose project contains a foreign container resource' "$foreign_error" \
|
||||||
|
|| task13_fail "stopped foreign container refusal was not explicit"
|
||||||
|
|
||||||
|
: >"$calls"
|
||||||
|
docker() {
|
||||||
|
printf '%s\n' "$*" >>"$calls"
|
||||||
|
case "$1 $2 $3" in
|
||||||
|
"container ls -aq") printf '%s\n' stopped-owned ;;
|
||||||
|
"container inspect --format") printf '%s\n' "$TASK13_RUN_ID" ;;
|
||||||
|
"volume ls -q"|"network ls -q") : ;;
|
||||||
|
*) return 1 ;;
|
||||||
|
esac
|
||||||
|
}
|
||||||
|
task13_assert_project_ownership
|
||||||
|
[[ "$(grep -Fc 'container inspect --format' "$calls")" -eq 1 ]] \
|
||||||
|
|| task13_fail "project cleanup did not inspect exactly the stopped owned container"
|
||||||
|
unset -f docker
|
||||||
|
rm -f "$calls" "$foreign_error"
|
||||||
|
}
|
||||||
|
|
||||||
|
task13_self_test_timeout_process_group() {
|
||||||
|
local child_file child_pid="" rc
|
||||||
|
child_file="$(mktemp "${TMPDIR:-/tmp}/thothii-task13-timeout-child.XXXXXX")"
|
||||||
|
set +e
|
||||||
|
task13_bounded 1 "child-process regression" bash -c \
|
||||||
|
'sleep 30 & printf "%s\n" "$!" >"$1"; wait' _ "$child_file" >/dev/null 2>&1
|
||||||
|
rc=$?
|
||||||
|
set -e
|
||||||
|
child_pid="$(sed -n '1p' "$child_file")"
|
||||||
|
[[ "$rc" -ne 0 ]] || {
|
||||||
|
rm -f "$child_file"
|
||||||
|
task13_fail "timed command unexpectedly succeeded"
|
||||||
|
}
|
||||||
|
if [[ -n "$child_pid" ]] && kill -0 "$child_pid" 2>/dev/null; then
|
||||||
|
kill -KILL "$child_pid" 2>/dev/null || true
|
||||||
|
rm -f "$child_file"
|
||||||
|
task13_fail "timed command left its child process alive"
|
||||||
|
fi
|
||||||
|
rm -f "$child_file"
|
||||||
|
}
|
||||||
|
|
||||||
|
task13_self_test_nested_timeout_process_group() {
|
||||||
|
local child_file child_pid="" rc
|
||||||
|
child_file="$(mktemp "${TMPDIR:-/tmp}/thothii-task13-nested-timeout.XXXXXX")"
|
||||||
|
task13_nested_timeout_fixture() {
|
||||||
|
task13_bounded 30 "nested child-process regression" bash -c \
|
||||||
|
'sleep 30 & printf "%s\n" "$!" >"$1"; wait' _ "$child_file"
|
||||||
|
}
|
||||||
|
set +e
|
||||||
|
task13_supervise 1 "nested timeout supervisor" task13_nested_timeout_fixture >/dev/null 2>&1
|
||||||
|
rc=$?
|
||||||
|
set -e
|
||||||
|
unset -f task13_nested_timeout_fixture
|
||||||
|
child_pid="$(sed -n '1p' "$child_file")"
|
||||||
|
[[ "$rc" -ne 0 ]] || {
|
||||||
|
rm -f "$child_file"
|
||||||
|
task13_fail "nested timed command unexpectedly succeeded"
|
||||||
|
}
|
||||||
|
if [[ -n "$child_pid" ]] && kill -0 "$child_pid" 2>/dev/null; then
|
||||||
|
kill -KILL "$child_pid" 2>/dev/null || true
|
||||||
|
rm -f "$child_file"
|
||||||
|
task13_fail "outer timeout left its nested command child alive"
|
||||||
|
fi
|
||||||
|
rm -f "$child_file"
|
||||||
|
}
|
||||||
|
|
||||||
|
task13_self_test_public_timeout_contract() {
|
||||||
|
local root
|
||||||
|
root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)"
|
||||||
|
grep -Eq 'task13_supervise[[:space:]].*task13_smoke_main[[:space:]]+full' \
|
||||||
|
"$root/scripts/unified-deployment-smoke.sh" \
|
||||||
|
|| task13_fail "direct unified smoke invocation lacks an internal supervisor"
|
||||||
|
grep -Eq 'task13_supervise[[:space:]].*task13_smoke_main[[:space:]]+update' \
|
||||||
|
"$root/scripts/thothctl-update-smoke.sh" \
|
||||||
|
|| task13_fail "direct update smoke invocation lacks an internal supervisor"
|
||||||
|
}
|
||||||
|
|
||||||
|
task13_self_test_windows_release_contract() {
|
||||||
|
local root script workflow
|
||||||
|
root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)"
|
||||||
|
script="$root/scripts/test-windows-clone-contract.ps1"
|
||||||
|
workflow="$root/.github/workflows/deployment.yml"
|
||||||
|
grep -Fq 'Task 13 path with spaces' "$script" \
|
||||||
|
|| task13_fail "Windows contract does not operate from a path containing spaces"
|
||||||
|
grep -Fq 'DockerStartup' "$script" \
|
||||||
|
|| task13_fail "Windows contract lacks an explicit Docker startup mode"
|
||||||
|
grep -Fq 'Kill($true)' "$script" \
|
||||||
|
|| task13_fail "Windows bounded runner does not kill the full process tree"
|
||||||
|
grep -Fq 'docker-desktop' "$workflow" \
|
||||||
|
|| task13_fail "workflow lacks a manual self-hosted Windows Docker Desktop gate"
|
||||||
|
grep -Fq -- '-DockerStartup' "$workflow" \
|
||||||
|
|| task13_fail "manual Windows release job does not execute Docker startup mode"
|
||||||
|
}
|
||||||
|
|
||||||
|
task13_self_test_server_release_contract() {
|
||||||
|
local root workflow server_smoke
|
||||||
|
root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)"
|
||||||
|
workflow="$root/.github/workflows/deployment.yml"
|
||||||
|
server_smoke="$root/scripts/server-deployment-smoke.sh"
|
||||||
|
[[ -x "$server_smoke" ]] || task13_fail "bounded Linux server deployment smoke is missing"
|
||||||
|
grep -Fq 'deploy/compose.server.yaml' "$root/scripts/unified-deployment-smoke.sh" \
|
||||||
|
|| task13_fail "server smoke does not load the server profile"
|
||||||
|
grep -Fq 'deploy/compose.session-server.yaml.example' "$root/scripts/unified-deployment-smoke.sh" \
|
||||||
|
|| task13_fail "server smoke does not load the required session overlay"
|
||||||
|
grep -Eq 'timeout .*scripts/server-deployment-smoke\.sh' "$workflow" \
|
||||||
|
|| task13_fail "workflow lacks an outer timeout for the Linux server smoke"
|
||||||
|
}
|
||||||
|
|
||||||
task13_self_test_source_contract() {
|
task13_self_test_source_contract() {
|
||||||
local root host_network push_command registry_function workflow uses_count pinned_uses_count
|
local root host_network push_command registry_function workflow uses_count pinned_uses_count
|
||||||
root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)"
|
root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)"
|
||||||
@@ -950,10 +1376,34 @@ task13_self_test() {
|
|||||||
task13_self_test_cleanup_ownership
|
task13_self_test_cleanup_ownership
|
||||||
task13_self_test_image_cleanup_ownership
|
task13_self_test_image_cleanup_ownership
|
||||||
task13_self_test_transaction_image_cleanup
|
task13_self_test_transaction_image_cleanup
|
||||||
|
task13_self_test_rollback_fixture_contract
|
||||||
|
task13_self_test_runtime_binding_fixture
|
||||||
|
task13_self_test_server_runtime_binding_fixture
|
||||||
|
task13_self_test_stopped_project_containers
|
||||||
|
task13_self_test_timeout_process_group
|
||||||
|
task13_self_test_nested_timeout_process_group
|
||||||
|
task13_self_test_public_timeout_contract
|
||||||
|
task13_self_test_windows_release_contract
|
||||||
|
task13_self_test_server_release_contract
|
||||||
task13_self_test_source_contract
|
task13_self_test_source_contract
|
||||||
printf 'Task 13 smoke safety contracts passed.\n'
|
printf 'Task 13 smoke safety contracts passed.\n'
|
||||||
}
|
}
|
||||||
|
|
||||||
|
task13_self_test_case() {
|
||||||
|
case "$1" in
|
||||||
|
rollback) task13_self_test_rollback_fixture_contract ;;
|
||||||
|
runtime-bindings) task13_self_test_runtime_binding_fixture ;;
|
||||||
|
server-bindings) task13_self_test_server_runtime_binding_fixture ;;
|
||||||
|
cleanup) task13_self_test_stopped_project_containers ;;
|
||||||
|
timeout-group) task13_self_test_timeout_process_group ;;
|
||||||
|
timeout-nested) task13_self_test_nested_timeout_process_group ;;
|
||||||
|
timeout-public) task13_self_test_public_timeout_contract ;;
|
||||||
|
windows) task13_self_test_windows_release_contract ;;
|
||||||
|
server) task13_self_test_server_release_contract ;;
|
||||||
|
*) task13_fail "unknown Task 13 self-test case: $1" ;;
|
||||||
|
esac
|
||||||
|
}
|
||||||
|
|
||||||
task13_initialize() {
|
task13_initialize() {
|
||||||
umask 077
|
umask 077
|
||||||
TASK13_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)"
|
TASK13_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)"
|
||||||
@@ -966,6 +1416,7 @@ task13_initialize() {
|
|||||||
trap 'task13_cleanup $?' EXIT
|
trap 'task13_cleanup $?' EXIT
|
||||||
trap 'exit 130' INT TERM HUP
|
trap 'exit 130' INT TERM HUP
|
||||||
TASK13_RUN_ID="$(date -u +%Y%m%d%H%M%S)-$$-${RANDOM:-0}"
|
TASK13_RUN_ID="$(date -u +%Y%m%d%H%M%S)-$$-${RANDOM:-0}"
|
||||||
|
TASK13_PROFILE="local"
|
||||||
TASK13_INSTALLATION="$TASK13_TMP/thothii-installation.yaml"
|
TASK13_INSTALLATION="$TASK13_TMP/thothii-installation.yaml"
|
||||||
TASK13_PROJECT="thothii-$(task13_sha256_text "$TASK13_INSTALLATION" | cut -c1-12)"
|
TASK13_PROJECT="thothii-$(task13_sha256_text "$TASK13_INSTALLATION" | cut -c1-12)"
|
||||||
TASK13_CONTROL_DIR="$TASK13_ROOT/.thothctl/$TASK13_PROJECT"
|
TASK13_CONTROL_DIR="$TASK13_ROOT/.thothctl/$TASK13_PROJECT"
|
||||||
@@ -984,12 +1435,22 @@ task13_initialize() {
|
|||||||
TASK13_LLM_SERVER="$TASK13_TMP/fake-llm.mjs"
|
TASK13_LLM_SERVER="$TASK13_TMP/fake-llm.mjs"
|
||||||
TASK13_THOTHCTL_DIR="$TASK13_TMP/thothctl"
|
TASK13_THOTHCTL_DIR="$TASK13_TMP/thothctl"
|
||||||
TASK13_LLM_CONTAINER="$TASK13_PROJECT-llm"
|
TASK13_LLM_CONTAINER="$TASK13_PROJECT-llm"
|
||||||
|
TASK13_BAD_CANDIDATE_CONTAINER="$TASK13_PROJECT-bad-candidate"
|
||||||
TASK13_CORE_IMAGE="task13-core-$TASK13_RUN_ID:local"
|
TASK13_CORE_IMAGE="task13-core-$TASK13_RUN_ID:local"
|
||||||
TASK13_FRONTEND_IMAGE="task13-frontend-$TASK13_RUN_ID:local"
|
TASK13_FRONTEND_IMAGE="task13-frontend-$TASK13_RUN_ID:local"
|
||||||
TASK13_SECRET_VALUE="task13-secret-$TASK13_RUN_ID"
|
TASK13_SECRET_VALUE="task13-secret-$TASK13_RUN_ID"
|
||||||
TASK13_NETWORK=""
|
TASK13_NETWORK=""
|
||||||
TASK13_BAD_CANDIDATE_ID=""
|
TASK13_BAD_CANDIDATE_ID=""
|
||||||
TASK13_PREVIOUS_IMAGE_ID=""
|
TASK13_PREVIOUS_IMAGE_ID=""
|
||||||
|
TASK13_SERVER_DATA="$TASK13_TMP/Server Data"
|
||||||
|
TASK13_SERVER_PI_STATE="$TASK13_TMP/Server Pi State"
|
||||||
|
TASK13_SERVER_REGISTRY="$TASK13_TMP/Server Registry"
|
||||||
|
TASK13_SERVER_WORKSPACE_CONFIG="$TASK13_TMP/server-sessions.yaml"
|
||||||
|
TASK13_SESSION_RUNTIME_PASSWORD="$TASK13_TMP/session-runtime-password"
|
||||||
|
TASK13_SESSION_MIGRATOR_PASSWORD_FILE="$TASK13_TMP/session-migrator-password"
|
||||||
|
TASK13_SESSION_CA="$TASK13_TMP/session-ca.pem"
|
||||||
|
TASK13_SESSION_PASSWORD="task13-runtime-$TASK13_RUN_ID"
|
||||||
|
TASK13_SESSION_MIGRATOR_PASSWORD="task13-migrator-$TASK13_RUN_ID"
|
||||||
}
|
}
|
||||||
|
|
||||||
task13_require_tools() {
|
task13_require_tools() {
|
||||||
@@ -1022,10 +1483,26 @@ task13_smoke_main() {
|
|||||||
printf 'Task 13 %s deployment smoke passed.\n' "$mode"
|
printf 'Task 13 %s deployment smoke passed.\n' "$mode"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
task13_server_smoke_main() {
|
||||||
|
task13_initialize
|
||||||
|
TASK13_PROFILE="server"
|
||||||
|
task13_require_tools
|
||||||
|
task13_write_server_fixture_files
|
||||||
|
task13_seed_registry
|
||||||
|
task13_start_server_stack
|
||||||
|
task13_assert_project_ownership
|
||||||
|
task13_assert_built_image_ownership
|
||||||
|
task13_assert_server_runtime
|
||||||
|
printf 'Task 13 Linux server deployment smoke passed.\n'
|
||||||
|
}
|
||||||
|
|
||||||
if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
|
if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
|
||||||
if [[ "${1:-}" == "--self-test" ]]; then
|
if [[ "${1:-}" == "--self-test" ]]; then
|
||||||
task13_self_test
|
task13_self_test
|
||||||
|
elif [[ "${1:-}" == "--self-test-case" ]]; then
|
||||||
|
[[ -n "${2:-}" ]] || task13_fail "--self-test-case requires a case name"
|
||||||
|
task13_self_test_case "$2"
|
||||||
else
|
else
|
||||||
task13_smoke_main full
|
task13_supervise "$TASK13_SMOKE_TIMEOUT" "unified deployment smoke" task13_smoke_main full
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
|
|||||||
@@ -193,6 +193,13 @@ type installationRunner struct {
|
|||||||
runner compose.Runner
|
runner compose.Runner
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (r installationRunner) SessionInventoryScope() string {
|
||||||
|
if r.installation.Profile == "local" {
|
||||||
|
return "mine"
|
||||||
|
}
|
||||||
|
return "all"
|
||||||
|
}
|
||||||
|
|
||||||
func (r installationRunner) Run(ctx context.Context, args []string, stdin io.Reader) (compose.Result, error) {
|
func (r installationRunner) Run(ctx context.Context, args []string, stdin io.Reader) (compose.Result, error) {
|
||||||
if len(args) > 0 && args[0] == "compose" {
|
if len(args) > 0 && args[0] == "compose" {
|
||||||
return r.runner.Run(ctx, r.installation.ComposeArgs(args[1:]...), stdin)
|
return r.runner.Run(ctx, r.installation.ComposeArgs(args[1:]...), stdin)
|
||||||
|
|||||||
@@ -526,8 +526,14 @@ func ensureMaintenance(ctx context.Context, runner Runner) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func activeSessions(ctx context.Context, runner Runner) (bool, error) {
|
func activeSessions(ctx context.Context, runner Runner) (bool, error) {
|
||||||
|
scope := "all"
|
||||||
|
if scoped, ok := runner.(interface{ SessionInventoryScope() string }); ok {
|
||||||
|
if requested := scoped.SessionInventoryScope(); requested == "mine" || requested == "all" {
|
||||||
|
scope = requested
|
||||||
|
}
|
||||||
|
}
|
||||||
args := append([]string{"exec", "-T", "core", "curl", "-fsS"}, internalIdentityHeaders...)
|
args := append([]string{"exec", "-T", "core", "curl", "-fsS"}, internalIdentityHeaders...)
|
||||||
args = append(args, "http://127.0.0.1:8787/sessions?scope=all")
|
args = append(args, "http://127.0.0.1:8787/sessions?scope="+scope)
|
||||||
result, err := runCompose(ctx, runner, args...)
|
result, err := runCompose(ctx, runner, args...)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, commandError("active-session check", result, err)
|
return false, commandError("active-session check", result, err)
|
||||||
|
|||||||
@@ -30,6 +30,30 @@ func TestActiveSessionsParsesAuthenticatedBackendBareArrayFixture(t *testing.T)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type scopedSessionRunner struct {
|
||||||
|
calls []string
|
||||||
|
scope string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *scopedSessionRunner) Run(_ context.Context, args []string, _ io.Reader) (compose.Result, error) {
|
||||||
|
r.calls = append(r.calls, strings.Join(args, " "))
|
||||||
|
return compose.Result{Stdout: "[]"}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *scopedSessionRunner) SessionInventoryScope() string { return r.scope }
|
||||||
|
|
||||||
|
func TestActiveSessionsUsesInstallationScopedInventory(t *testing.T) {
|
||||||
|
for _, scope := range []string{"mine", "all"} {
|
||||||
|
runner := &scopedSessionRunner{scope: scope}
|
||||||
|
if active, err := activeSessions(context.Background(), runner); err != nil || active {
|
||||||
|
t.Fatalf("activeSessions(%s) = %t, %v; want false, nil", scope, active, err)
|
||||||
|
}
|
||||||
|
if len(runner.calls) != 1 || !strings.Contains(runner.calls[0], "/sessions?scope="+scope) {
|
||||||
|
t.Fatalf("activeSessions(%s) call = %v; want installation-scoped inventory", scope, runner.calls)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestUpdateBuildsPinnedVersionRecreatesOnlyCoreAndPersistsRecoveryState(t *testing.T) {
|
func TestUpdateBuildsPinnedVersionRecreatesOnlyCoreAndPersistsRecoveryState(t *testing.T) {
|
||||||
fake := newFakeRunner()
|
fake := newFakeRunner()
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
|
|||||||
Reference in New Issue
Block a user