fix: complete deployment release gates
This commit is contained in:
Executable
+8
@@ -0,0 +1,8 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
root="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||
# shellcheck source=./unified-deployment-smoke.sh
|
||||
source "$root/scripts/unified-deployment-smoke.sh"
|
||||
|
||||
task13_supervise "$TASK13_SMOKE_TIMEOUT" "Linux server deployment smoke" task13_server_smoke_main
|
||||
@@ -1,27 +1,90 @@
|
||||
param(
|
||||
[string]$RepositoryRoot = "",
|
||||
[string]$ThothctlPath = ""
|
||||
[switch]$DockerStartup,
|
||||
[int]$CommandTimeoutSeconds = 600
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
Set-StrictMode -Version Latest
|
||||
$script:SensitiveValues = [System.Collections.Generic.List[string]]::new()
|
||||
|
||||
function Protect-Output([string]$Value) {
|
||||
$protected = $Value
|
||||
foreach ($secret in $script:SensitiveValues) {
|
||||
if (-not [string]::IsNullOrEmpty($secret)) {
|
||||
$protected = $protected.Replace($secret, "[REDACTED]")
|
||||
}
|
||||
}
|
||||
return $protected -replace '(?i)((?:password|token|api[_-]?key|secret|key)\s*[:=]\s*)[^\s,;]+', '$1[REDACTED]'
|
||||
}
|
||||
|
||||
function Invoke-BoundedNative {
|
||||
param(
|
||||
[Parameter(Mandatory = $true)][string]$FilePath,
|
||||
[Parameter(Mandatory = $true)][string[]]$Arguments,
|
||||
[Parameter(Mandatory = $true)][string]$Label,
|
||||
[string]$WorkingDirectory = "",
|
||||
[int]$TimeoutSeconds = $CommandTimeoutSeconds,
|
||||
[switch]$AllowFailure
|
||||
)
|
||||
$startInfo = [System.Diagnostics.ProcessStartInfo]::new()
|
||||
$startInfo.FileName = $FilePath
|
||||
$startInfo.UseShellExecute = $false
|
||||
$startInfo.RedirectStandardOutput = $true
|
||||
$startInfo.RedirectStandardError = $true
|
||||
$startInfo.CreateNoWindow = $true
|
||||
if (-not [string]::IsNullOrWhiteSpace($WorkingDirectory)) {
|
||||
$startInfo.WorkingDirectory = $WorkingDirectory
|
||||
}
|
||||
foreach ($argument in $Arguments) {
|
||||
[void]$startInfo.ArgumentList.Add($argument)
|
||||
}
|
||||
$process = [System.Diagnostics.Process]::new()
|
||||
$process.StartInfo = $startInfo
|
||||
if (-not $process.Start()) {
|
||||
throw "$Label could not start"
|
||||
}
|
||||
$stdoutTask = $process.StandardOutput.ReadToEndAsync()
|
||||
$stderrTask = $process.StandardError.ReadToEndAsync()
|
||||
if (-not $process.WaitForExit($TimeoutSeconds * 1000)) {
|
||||
$process.Kill($true)
|
||||
[void]$process.WaitForExit(30000)
|
||||
throw "$Label timed out after $TimeoutSeconds seconds"
|
||||
}
|
||||
$stdout = $stdoutTask.GetAwaiter().GetResult()
|
||||
$stderr = $stderrTask.GetAwaiter().GetResult()
|
||||
$result = [pscustomobject]@{
|
||||
ExitCode = $process.ExitCode
|
||||
StdOut = $stdout
|
||||
StdErr = $stderr
|
||||
}
|
||||
if (-not $AllowFailure -and $result.ExitCode -ne 0) {
|
||||
$diagnostic = Protect-Output (($result.StdOut + "`n" + $result.StdErr).Trim())
|
||||
throw "$Label failed with exit $($result.ExitCode): $diagnostic"
|
||||
}
|
||||
return $result
|
||||
}
|
||||
|
||||
function Write-Utf8File([string]$Path, [string]$Contents) {
|
||||
[System.IO.Directory]::CreateDirectory([System.IO.Path]::GetDirectoryName($Path)) | Out-Null
|
||||
[System.IO.File]::WriteAllText($Path, $Contents, [System.Text.UTF8Encoding]::new($false))
|
||||
}
|
||||
|
||||
function ConvertTo-YamlPath([string]$Path) {
|
||||
return $Path.Replace('\', '/').Replace('"', '\"')
|
||||
}
|
||||
|
||||
if ([string]::IsNullOrWhiteSpace($RepositoryRoot)) {
|
||||
$RepositoryRoot = (& git rev-parse --show-toplevel).Trim()
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "git could not resolve the repository root"
|
||||
}
|
||||
$resolved = Invoke-BoundedNative -FilePath "git" -Arguments @("rev-parse", "--show-toplevel") -Label "resolve repository root"
|
||||
$RepositoryRoot = $resolved.StdOut.Trim()
|
||||
}
|
||||
$RepositoryRoot = [System.IO.Path]::GetFullPath($RepositoryRoot)
|
||||
|
||||
$tracked = @(& git -C $RepositoryRoot ls-files)
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "git ls-files failed"
|
||||
}
|
||||
|
||||
$trackedResult = Invoke-BoundedNative -FilePath "git" -Arguments @("-C", $RepositoryRoot, "ls-files") -Label "list tracked files"
|
||||
$tracked = @($trackedResult.StdOut -split "`r?`n" | Where-Object { -not [string]::IsNullOrWhiteSpace($_) })
|
||||
$scriptRelativePath = "scripts/test-windows-clone-contract.ps1"
|
||||
$eolAttribute = (& git -C $RepositoryRoot check-attr eol -- $scriptRelativePath).Trim()
|
||||
if ($LASTEXITCODE -ne 0 -or -not $eolAttribute.EndsWith("eol: crlf", [System.StringComparison]::OrdinalIgnoreCase)) {
|
||||
$attribute = Invoke-BoundedNative -FilePath "git" -Arguments @("-C", $RepositoryRoot, "check-attr", "eol", "--", $scriptRelativePath) -Label "read PowerShell eol attribute"
|
||||
if (-not $attribute.StdOut.Trim().EndsWith("eol: crlf", [System.StringComparison]::OrdinalIgnoreCase)) {
|
||||
throw "the Windows contract script must have the repository eol=crlf attribute"
|
||||
}
|
||||
$scriptBytes = [System.IO.File]::ReadAllBytes((Join-Path $RepositoryRoot $scriptRelativePath))
|
||||
@@ -38,71 +101,203 @@ foreach ($relativePath in $tracked) {
|
||||
$name.Equals("Dockerfile", [System.StringComparison]::OrdinalIgnoreCase) -or
|
||||
$name.StartsWith("Dockerfile.", [System.StringComparison]::OrdinalIgnoreCase) -or
|
||||
$name.EndsWith(".Dockerfile", [System.StringComparison]::OrdinalIgnoreCase)
|
||||
if (-not $mustBeLf) {
|
||||
continue
|
||||
}
|
||||
$absolutePath = Join-Path $RepositoryRoot $relativePath
|
||||
$bytes = [System.IO.File]::ReadAllBytes($absolutePath)
|
||||
if ($bytes -contains [byte]0x0D) {
|
||||
$offenders.Add($relativePath)
|
||||
if ($mustBeLf) {
|
||||
$bytes = [System.IO.File]::ReadAllBytes((Join-Path $RepositoryRoot $relativePath))
|
||||
if ($bytes -contains [byte]0x0D) {
|
||||
$offenders.Add($relativePath)
|
||||
}
|
||||
}
|
||||
}
|
||||
if ($offenders.Count -ne 0) {
|
||||
throw "CR byte 0x0D found in tracked LF contract files: $($offenders -join ', ')"
|
||||
}
|
||||
|
||||
$temporaryRoot = Join-Path ([System.IO.Path]::GetTempPath()) ("thothii-windows-contract-" + [guid]::NewGuid().ToString("N"))
|
||||
$savedEnvironment = @{
|
||||
THT_WORKSPACE_GIT_REMOTE = $env:THT_WORKSPACE_GIT_REMOTE
|
||||
PI_AUTH_FILE = $env:PI_AUTH_FILE
|
||||
THT_SECRETS_FILE = $env:THT_SECRETS_FILE
|
||||
}
|
||||
try {
|
||||
[System.IO.Directory]::CreateDirectory($temporaryRoot) | Out-Null
|
||||
$piAuth = Join-Path $temporaryRoot "pi-auth.json"
|
||||
$secrets = Join-Path $temporaryRoot "thothii.secrets"
|
||||
[System.IO.File]::WriteAllText($piAuth, "{}`n", [System.Text.UTF8Encoding]::new($false))
|
||||
[System.IO.File]::WriteAllText($secrets, "THT_MODEL_API_KEY=windows-contract`n", [System.Text.UTF8Encoding]::new($false))
|
||||
$runId = [guid]::NewGuid().ToString("N")
|
||||
$temporaryRoot = Join-Path ([System.IO.Path]::GetTempPath()) ("ThothII Task 13 path with spaces " + $runId)
|
||||
$spacedRepository = Join-Path $temporaryRoot "Task 13 path with spaces"
|
||||
$fixtureRoot = Join-Path $temporaryRoot "Disposable Fixture Data"
|
||||
$project = "thothii-win-" + $runId.Substring(0, 12)
|
||||
$runLabel = "windows-" + $runId
|
||||
$coreImage = "task13-windows-core-$($runId.Substring(0, 16)):local"
|
||||
$frontendImage = "task13-windows-frontend-$($runId.Substring(0, 16)):local"
|
||||
$composeArguments = @()
|
||||
$startupAttempted = $false
|
||||
$cleanupSucceeded = $true
|
||||
$savedEnvironment = @{}
|
||||
|
||||
$env:THT_WORKSPACE_GIT_REMOTE = "https://git.example.invalid/platform/thoth-workspaces.git"
|
||||
$env:PI_AUTH_FILE = $piAuth
|
||||
$env:THT_SECRETS_FILE = $secrets
|
||||
$composeFiles = @(
|
||||
"--project-directory", $RepositoryRoot,
|
||||
"-f", (Join-Path $RepositoryRoot "compose.yaml"),
|
||||
"-f", (Join-Path $RepositoryRoot "deploy/compose.local.yaml")
|
||||
)
|
||||
$services = @(& docker compose @composeFiles config --services)
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "Docker Compose could not render the Windows clone"
|
||||
try {
|
||||
[System.IO.Directory]::CreateDirectory($spacedRepository) | Out-Null
|
||||
foreach ($relativePath in $tracked) {
|
||||
$source = Join-Path $RepositoryRoot $relativePath
|
||||
$destination = Join-Path $spacedRepository $relativePath
|
||||
[System.IO.Directory]::CreateDirectory([System.IO.Path]::GetDirectoryName($destination)) | Out-Null
|
||||
Copy-Item -LiteralPath $source -Destination $destination
|
||||
}
|
||||
if ((($services | Sort-Object) -join ",") -ne "core,frontend") {
|
||||
|
||||
$thothctl = Join-Path $spacedRepository "dist/thothctl/thothctl-windows-amd64.exe"
|
||||
[System.IO.Directory]::CreateDirectory([System.IO.Path]::GetDirectoryName($thothctl)) | Out-Null
|
||||
Invoke-BoundedNative -FilePath "go" -Arguments @("build", "-trimpath", "-o", $thothctl, "./cmd/thothctl") `
|
||||
-WorkingDirectory (Join-Path $spacedRepository "tools/thothctl") -Label "build native Windows thothctl in spaced path" | Out-Null
|
||||
Invoke-BoundedNative -FilePath $thothctl -Arguments @("--help") -Label "invoke native Windows thothctl from spaced path" | Out-Null
|
||||
|
||||
$piAuth = Join-Path $fixtureRoot "Pi Auth/pi-auth.json"
|
||||
$secrets = Join-Path $fixtureRoot "Secrets/thothii.secrets"
|
||||
$secretValue = "windows-contract-$runId"
|
||||
$script:SensitiveValues.Add($secretValue)
|
||||
Write-Utf8File $piAuth "{}`n"
|
||||
Write-Utf8File $secrets "THT_MODEL_API_KEY=$secretValue`n"
|
||||
|
||||
$remote = Join-Path $fixtureRoot "Workspace Remote/remote.git"
|
||||
$seed = Join-Path $fixtureRoot "Workspace Seed"
|
||||
[System.IO.Directory]::CreateDirectory((Join-Path $seed "workspaces")) | Out-Null
|
||||
Invoke-BoundedNative -FilePath "git" -Arguments @("init", "--bare", "--initial-branch=main", $remote) -Label "initialize Windows bare registry" | Out-Null
|
||||
Invoke-BoundedNative -FilePath "git" -Arguments @("-C", $seed, "init", "--initial-branch=main") -Label "initialize Windows registry seed" | Out-Null
|
||||
Write-Utf8File (Join-Path $seed "workspaces/task13-windows.yaml") @"
|
||||
workspace:
|
||||
schema_version: 2
|
||||
id: task13-windows
|
||||
name: Task 13 Windows
|
||||
language: en
|
||||
dwh:
|
||||
engine: postgres
|
||||
database: warehouse
|
||||
schema: public
|
||||
supported_transports: [postgres_direct]
|
||||
"@
|
||||
Invoke-BoundedNative -FilePath "git" -Arguments @("-C", $seed, "add", "workspaces/task13-windows.yaml") -Label "stage Windows registry seed" | Out-Null
|
||||
Invoke-BoundedNative -FilePath "git" -Arguments @("-C", $seed, "-c", "user.name=Task 13 Windows", "-c", "user.email=task13-windows@example.invalid", "commit", "-m", "Seed Windows smoke") -Label "commit Windows registry seed" | Out-Null
|
||||
Invoke-BoundedNative -FilePath "git" -Arguments @("-C", $seed, "push", $remote, "HEAD:main") -Label "push Windows registry seed" | Out-Null
|
||||
|
||||
$envFile = Join-Path $fixtureRoot "Config/local.env"
|
||||
$override = Join-Path $fixtureRoot "Config/compose.windows.yaml"
|
||||
$installation = Join-Path $fixtureRoot "Config/thothii installation.yaml"
|
||||
Write-Utf8File $envFile @"
|
||||
THOTH_HTTP_PORT=0
|
||||
THOTH_CORE_HTTP_PORT=0
|
||||
PI_AUTH_FILE=$piAuth
|
||||
THT_SECRETS_FILE=$secrets
|
||||
THT_WORKSPACE_GIT_REMOTE=/fixtures/remote.git
|
||||
THT_WORKSPACE_GIT_BRANCH=main
|
||||
THT_WORKSPACE_INSTALLATION_ID=task13-windows
|
||||
"@
|
||||
$remoteYaml = ConvertTo-YamlPath $remote
|
||||
Write-Utf8File $override @"
|
||||
services:
|
||||
core:
|
||||
image: $coreImage
|
||||
build:
|
||||
labels:
|
||||
io.thothii.task13.run: "$runLabel"
|
||||
labels:
|
||||
io.thothii.task13.run: "$runLabel"
|
||||
volumes:
|
||||
- "$remoteYaml:/fixtures/remote.git:ro"
|
||||
frontend:
|
||||
image: $frontendImage
|
||||
build:
|
||||
labels:
|
||||
io.thothii.task13.run: "$runLabel"
|
||||
labels:
|
||||
io.thothii.task13.run: "$runLabel"
|
||||
networks:
|
||||
thothii:
|
||||
labels:
|
||||
io.thothii.task13.run: "$runLabel"
|
||||
volumes:
|
||||
settings:
|
||||
labels:
|
||||
io.thothii.task13.run: "$runLabel"
|
||||
pi-state:
|
||||
labels:
|
||||
io.thothii.task13.run: "$runLabel"
|
||||
workspace-registry:
|
||||
labels:
|
||||
io.thothii.task13.run: "$runLabel"
|
||||
sessions:
|
||||
labels:
|
||||
io.thothii.task13.run: "$runLabel"
|
||||
"@
|
||||
$repoYaml = ConvertTo-YamlPath $spacedRepository
|
||||
$envYaml = ConvertTo-YamlPath $envFile
|
||||
$overrideYaml = ConvertTo-YamlPath $override
|
||||
Write-Utf8File $installation @"
|
||||
profile: local
|
||||
projectDirectory: "$repoYaml"
|
||||
envFile: "$envYaml"
|
||||
overrides:
|
||||
- "$overrideYaml"
|
||||
"@
|
||||
|
||||
$composeArguments = @(
|
||||
"compose", "--project-name", $project, "--project-directory", $spacedRepository,
|
||||
"--env-file", $envFile,
|
||||
"-f", (Join-Path $spacedRepository "compose.yaml"),
|
||||
"-f", (Join-Path $spacedRepository "deploy/compose.local.yaml"),
|
||||
"-f", $override
|
||||
)
|
||||
$render = Invoke-BoundedNative -FilePath "docker" -Arguments ($composeArguments + @("config", "--services")) -Label "render Windows Compose from spaced path"
|
||||
$services = @($render.StdOut -split "`r?`n" | Where-Object { $_ } | Sort-Object)
|
||||
if (($services -join ",") -ne "core,frontend") {
|
||||
throw "rendered Windows stack must contain exactly core and frontend"
|
||||
}
|
||||
& docker compose @composeFiles config --quiet
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "Docker Compose rejected the Windows clone"
|
||||
}
|
||||
Invoke-BoundedNative -FilePath "docker" -Arguments ($composeArguments + @("config", "--quiet")) -Label "validate Windows Compose from spaced path" | Out-Null
|
||||
|
||||
if ([string]::IsNullOrWhiteSpace($ThothctlPath)) {
|
||||
$ThothctlPath = Join-Path $RepositoryRoot "dist/thothctl/thothctl-windows-amd64.exe"
|
||||
}
|
||||
$ThothctlPath = [System.IO.Path]::GetFullPath($ThothctlPath)
|
||||
if (-not [System.IO.File]::Exists($ThothctlPath)) {
|
||||
throw "Windows thothctl binary is missing: $ThothctlPath"
|
||||
}
|
||||
& $ThothctlPath --help | Out-Null
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "Windows thothctl invocation failed"
|
||||
if ($DockerStartup) {
|
||||
Invoke-BoundedNative -FilePath "docker" -Arguments @("info") -Label "verify Windows Docker Desktop readiness" -TimeoutSeconds 60 | Out-Null
|
||||
$startupAttempted = $true
|
||||
Invoke-BoundedNative -FilePath "docker" -Arguments ($composeArguments + @("build", "--pull")) -Label "build two-service Windows stack" | Out-Null
|
||||
Invoke-BoundedNative -FilePath "docker" -Arguments ($composeArguments + @("up", "--detach", "--wait", "--wait-timeout", "180")) -Label "start two-service Windows stack" -TimeoutSeconds 300 | Out-Null
|
||||
$running = Invoke-BoundedNative -FilePath "docker" -Arguments ($composeArguments + @("ps", "--status", "running", "--services")) -Label "inspect running Windows services"
|
||||
$runningServices = @($running.StdOut -split "`r?`n" | Where-Object { $_ } | Sort-Object)
|
||||
if (($runningServices -join ",") -ne "core,frontend") {
|
||||
throw "bounded Windows startup did not leave exactly core and frontend running"
|
||||
}
|
||||
Invoke-BoundedNative -FilePath $thothctl -Arguments @("--installation", $installation, "status") -Label "invoke installation-aware Windows thothctl in spaced path" | Out-Null
|
||||
}
|
||||
}
|
||||
finally {
|
||||
if ($startupAttempted -and $composeArguments.Count -ne 0) {
|
||||
try {
|
||||
foreach ($kind in @("container", "volume", "network")) {
|
||||
$listArgs = if ($kind -eq "container") { @($kind, "ls", "-aq") } else { @($kind, "ls", "-q") }
|
||||
$listed = Invoke-BoundedNative -FilePath "docker" -Arguments ($listArgs + @("--filter", "label=com.docker.compose.project=$project")) -Label "enumerate Windows project $kind resources" -TimeoutSeconds 30
|
||||
foreach ($id in @($listed.StdOut -split "`r?`n" | Where-Object { $_ })) {
|
||||
$format = if ($kind -eq "container") { '{{ index .Config.Labels "io.thothii.task13.run" }}' } else { '{{ index .Labels "io.thothii.task13.run" }}' }
|
||||
$inspected = Invoke-BoundedNative -FilePath "docker" -Arguments @($kind, "inspect", "--format", $format, $id) -Label "inspect Windows project $kind ownership" -TimeoutSeconds 30
|
||||
if ($inspected.StdOut.Trim() -ne $runLabel) {
|
||||
throw "refusing to remove foreign Windows project $kind resource"
|
||||
}
|
||||
}
|
||||
}
|
||||
Invoke-BoundedNative -FilePath "docker" -Arguments ($composeArguments + @("down", "--volumes", "--remove-orphans", "--timeout", "10")) -Label "remove exact Windows Compose project" -TimeoutSeconds 60 | Out-Null
|
||||
foreach ($image in @($frontendImage, $coreImage)) {
|
||||
$inspection = Invoke-BoundedNative -FilePath "docker" -Arguments @("image", "inspect", "--format", '{{ index .Config.Labels "io.thothii.task13.run" }}', $image) -Label "inspect Windows image ownership" -TimeoutSeconds 30 -AllowFailure
|
||||
if ($inspection.ExitCode -eq 0) {
|
||||
if ($inspection.StdOut.Trim() -ne $runLabel) {
|
||||
throw "refusing to remove foreign Windows image $image"
|
||||
}
|
||||
Invoke-BoundedNative -FilePath "docker" -Arguments @("image", "rm", $image) -Label "remove exact Windows image" -TimeoutSeconds 60 | Out-Null
|
||||
}
|
||||
}
|
||||
}
|
||||
catch {
|
||||
$cleanupSucceeded = $false
|
||||
Write-Error (Protect-Output $_.Exception.Message)
|
||||
}
|
||||
}
|
||||
foreach ($name in $savedEnvironment.Keys) {
|
||||
[System.Environment]::SetEnvironmentVariable($name, $savedEnvironment[$name], "Process")
|
||||
}
|
||||
if ([System.IO.Directory]::Exists($temporaryRoot)) {
|
||||
if ($cleanupSucceeded -and [System.IO.Directory]::Exists($temporaryRoot)) {
|
||||
Remove-Item -LiteralPath $temporaryRoot -Recurse -Force
|
||||
}
|
||||
}
|
||||
|
||||
Write-Output "Windows clone, LF-byte, Compose render, and thothctl invocation contracts passed."
|
||||
if (-not $cleanupSucceeded) {
|
||||
throw "Windows cleanup proof failed; fixture path retained for recovery"
|
||||
}
|
||||
if ($DockerStartup) {
|
||||
Write-Output "Windows spaced-path build, native thothctl, bounded two-service startup, and exact cleanup passed."
|
||||
} else {
|
||||
Write-Output "Windows spaced-path clone, LF-byte, Compose render, and native thothctl build/invocation contracts passed; Docker startup mode was not requested."
|
||||
}
|
||||
|
||||
@@ -5,4 +5,4 @@ root="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||
# shellcheck source=./unified-deployment-smoke.sh
|
||||
source "$root/scripts/unified-deployment-smoke.sh"
|
||||
|
||||
task13_smoke_main update
|
||||
task13_supervise "$TASK13_SMOKE_TIMEOUT" "thothctl update smoke" task13_smoke_main update
|
||||
|
||||
@@ -4,11 +4,15 @@
|
||||
# isolated fixture, exact cleanup, and sanitized failure reporting.
|
||||
set -euo pipefail
|
||||
|
||||
TASK13_BAD_CANDIDATE_IMAGE="registry:2.8.3@sha256:a3d8aaa63ed8681a604f1dea0aa03f100d5895b6a58ace528858a7b332415373"
|
||||
TASK13_BAD_CANDIDATE_IMAGE="hello-world@sha256:c3cbe1cc1aa588a64951ac6286e0df7b27fe2e6324b1001c619bb358770c0178"
|
||||
TASK13_BAD_CANDIDATE_BEHAVIOR="stopped"
|
||||
TASK13_BAD_PI_VERSION="0.80.4-task13"
|
||||
TASK13_CURL_CONNECT_TIMEOUT=3
|
||||
TASK13_CURL_MAX_TIME=10
|
||||
TASK13_CLEANUP_TIMEOUT=20
|
||||
TASK13_COMMAND_TIMEOUT=900
|
||||
TASK13_SMOKE_TIMEOUT=1800
|
||||
TASK13_TERM_GRACE=45
|
||||
|
||||
task13_fail() {
|
||||
printf 'Task 13 smoke failed: %s\n' "$*" >&2
|
||||
@@ -49,41 +53,82 @@ task13_log_failure() {
|
||||
task13_run_logged() {
|
||||
local label="$1"
|
||||
shift
|
||||
if ! "$@" >>"$TASK13_LOG" 2>&1; then
|
||||
if ! task13_bounded "$TASK13_COMMAND_TIMEOUT" "$label" "$@" >>"$TASK13_LOG" 2>&1; then
|
||||
task13_log_failure "$label"
|
||||
fi
|
||||
}
|
||||
|
||||
task13_bounded() {
|
||||
local seconds="$1" label="$2" command_pid watchdog_pid rc
|
||||
local seconds="$1" label="$2" command_pid watchdog_pid rc watchdog_rc=0
|
||||
local monitor_enabled=0 timeout_marker command_group
|
||||
shift 2
|
||||
"$@" &
|
||||
command_pid=$!
|
||||
timeout_marker="$(mktemp "${TMPDIR:-/tmp}/thothii-task13-timeout.XXXXXX")"
|
||||
[[ $- == *m* ]] && monitor_enabled=1
|
||||
if [[ -n "${TASK13_ACTIVE_GROUP:-}" ]]; then
|
||||
[[ "$TASK13_ACTIVE_GROUP" =~ ^[0-9]+$ ]] \
|
||||
|| task13_fail "invalid active Task 13 process group"
|
||||
set +m
|
||||
"$@" &
|
||||
command_pid=$!
|
||||
command_group="$TASK13_ACTIVE_GROUP"
|
||||
else
|
||||
set -m
|
||||
"$@" &
|
||||
command_pid=$!
|
||||
command_group="$command_pid"
|
||||
[[ "$monitor_enabled" -eq 1 ]] || set +m
|
||||
fi
|
||||
(
|
||||
local sleep_pid
|
||||
local sleep_pid grace_deadline
|
||||
sleep "$seconds" &
|
||||
sleep_pid=$!
|
||||
trap 'kill "$sleep_pid" 2>/dev/null || true' EXIT INT TERM
|
||||
wait "$sleep_pid" 2>/dev/null || exit 0
|
||||
trap - EXIT INT TERM
|
||||
if kill -0 "$command_pid" 2>/dev/null; then
|
||||
if kill -0 -- "-$command_group" 2>/dev/null; then
|
||||
trap '' TERM
|
||||
printf 'timeout\n' >"$timeout_marker"
|
||||
printf 'Task 13 command timed out after %ss: %s\n' "$seconds" "$label" >&2
|
||||
kill -TERM "$command_pid" 2>/dev/null || true
|
||||
sleep 5
|
||||
kill -KILL "$command_pid" 2>/dev/null || true
|
||||
kill -TERM -- "-$command_group" 2>/dev/null || true
|
||||
grace_deadline=$((SECONDS + TASK13_TERM_GRACE))
|
||||
while kill -0 -- "-$command_group" 2>/dev/null && ((SECONDS < grace_deadline)); do
|
||||
sleep 1
|
||||
done
|
||||
kill -KILL -- "-$command_group" 2>/dev/null || true
|
||||
exit 124
|
||||
fi
|
||||
) &
|
||||
watchdog_pid=$!
|
||||
if [[ -n "${TASK13_ACTIVE_GROUP:-}" && "$monitor_enabled" -eq 1 ]]; then
|
||||
set -m
|
||||
fi
|
||||
if wait "$command_pid"; then
|
||||
rc=0
|
||||
else
|
||||
rc=$?
|
||||
fi
|
||||
kill "$watchdog_pid" 2>/dev/null || true
|
||||
wait "$watchdog_pid" 2>/dev/null || true
|
||||
if [[ -s "$timeout_marker" ]]; then
|
||||
wait "$watchdog_pid" 2>/dev/null || watchdog_rc=$?
|
||||
else
|
||||
kill "$watchdog_pid" 2>/dev/null || true
|
||||
wait "$watchdog_pid" 2>/dev/null || true
|
||||
fi
|
||||
rm -f "$timeout_marker"
|
||||
[[ "$watchdog_rc" -eq 124 ]] && return 124
|
||||
return "$rc"
|
||||
}
|
||||
|
||||
task13_supervised_call() {
|
||||
TASK13_ACTIVE_GROUP="$BASHPID"
|
||||
"$@"
|
||||
}
|
||||
|
||||
task13_supervise() {
|
||||
local seconds="$1" label="$2"
|
||||
shift 2
|
||||
task13_bounded "$seconds" "$label" task13_supervised_call "$@"
|
||||
}
|
||||
|
||||
task13_compose_files() {
|
||||
TASK13_COMPOSE=(
|
||||
docker compose
|
||||
@@ -91,10 +136,20 @@ task13_compose_files() {
|
||||
--project-directory "$TASK13_ROOT"
|
||||
--env-file "$TASK13_ENV_FILE"
|
||||
-f "$TASK13_ROOT/compose.yaml"
|
||||
-f "$TASK13_ROOT/deploy/compose.local.yaml"
|
||||
-f "$TASK13_OVERRIDE"
|
||||
)
|
||||
if [[ -f "$TASK13_CURRENT_IMAGE_OVERRIDE" ]]; then
|
||||
if [[ "${TASK13_PROFILE:-local}" == server ]]; then
|
||||
TASK13_COMPOSE+=(
|
||||
-f "$TASK13_ROOT/deploy/compose.server.yaml"
|
||||
-f "$TASK13_ROOT/deploy/compose.session-server.yaml.example"
|
||||
-f "$TASK13_OVERRIDE"
|
||||
)
|
||||
else
|
||||
TASK13_COMPOSE+=(
|
||||
-f "$TASK13_ROOT/deploy/compose.local.yaml"
|
||||
-f "$TASK13_OVERRIDE"
|
||||
)
|
||||
fi
|
||||
if [[ "${TASK13_PROFILE:-local}" == local && -f "$TASK13_CURRENT_IMAGE_OVERRIDE" ]]; then
|
||||
TASK13_COMPOSE+=(-f "$TASK13_CURRENT_IMAGE_OVERRIDE")
|
||||
fi
|
||||
}
|
||||
@@ -224,6 +279,15 @@ services:
|
||||
PI_THINKING: low
|
||||
THT_WORKSPACE_INSTALLATION_ID: task13-smoke
|
||||
THT_WORKSPACE_REGISTRY_ROOT: /data/workspace-registry
|
||||
THT_WS_TASK13_SMOKE_DWH_HOST: dwh.task13.invalid
|
||||
THT_WS_TASK13_SMOKE_DWH_PORT: "5432"
|
||||
THT_WS_TASK13_SMOKE_DWH_USER: task13_reader
|
||||
THT_WS_TASK13_SMOKE_DWH_PASSWORD_FILE: /run/secrets/thothii.secrets
|
||||
THT_WS_TASK13_SMOKE_VECTOR_HOST: vector.task13.invalid
|
||||
THT_WS_TASK13_SMOKE_VECTOR_PORT: "5432"
|
||||
THT_WS_TASK13_SMOKE_VECTOR_USER: task13_vector_reader
|
||||
THT_WS_TASK13_SMOKE_VECTOR_PASSWORD_FILE: /run/secrets/thothii.secrets
|
||||
THT_WS_TASK13_SMOKE_EMBEDDING_BASE_URL: http://$TASK13_LLM_CONTAINER:9000
|
||||
labels:
|
||||
io.thothii.task13.run: "$TASK13_RUN_ID"
|
||||
volumes: !override
|
||||
@@ -272,6 +336,114 @@ EOF
|
||||
chmod 0600 "$TASK13_INSTALLATION"
|
||||
}
|
||||
|
||||
task13_write_server_fixture_files() {
|
||||
local data_root pi_root registry_root remote_path workspace_path
|
||||
printf '{}\n' >"$TASK13_PI_AUTH"
|
||||
printf 'THT_MODEL_API_KEY=%s\n' "$TASK13_SECRET_VALUE" >"$TASK13_SECRETS"
|
||||
printf '%s\n' "$TASK13_SESSION_PASSWORD" >"$TASK13_SESSION_RUNTIME_PASSWORD"
|
||||
printf '%s\n' "$TASK13_SESSION_MIGRATOR_PASSWORD" >"$TASK13_SESSION_MIGRATOR_PASSWORD_FILE"
|
||||
cat >"$TASK13_SESSION_CA" <<'EOF'
|
||||
-----BEGIN CERTIFICATE-----
|
||||
VEFTSzEzLURJU1BPU0FCTEUtU0VTU0lPTi1DQQ==
|
||||
-----END CERTIFICATE-----
|
||||
EOF
|
||||
chmod 0644 "$TASK13_PI_AUTH" "$TASK13_SESSION_CA"
|
||||
chmod 0600 "$TASK13_SECRETS" "$TASK13_SESSION_RUNTIME_PASSWORD" \
|
||||
"$TASK13_SESSION_MIGRATOR_PASSWORD_FILE"
|
||||
|
||||
cat >"$TASK13_SERVER_WORKSPACE_CONFIG" <<'EOF'
|
||||
language: en
|
||||
session_storage:
|
||||
type: postgres_direct
|
||||
connection:
|
||||
host: ${THT_SESSION_DB_HOST}
|
||||
port: ${THT_SESSION_DB_PORT}
|
||||
database: ${THT_SESSION_DB_NAME}
|
||||
schema: thoth_sessions
|
||||
user: ${THT_SESSION_RUNTIME_USER}
|
||||
password_file: ${THT_SESSION_RUNTIME_PASSWORD_FILE}
|
||||
sslmode: ${THT_SESSION_DB_SSLMODE}
|
||||
sslrootcert: ${THT_SESSION_DB_SSLROOTCERT}
|
||||
roots:
|
||||
artifacts: artifacts
|
||||
indexes: indexes
|
||||
sessions: sessions
|
||||
EOF
|
||||
chmod 0600 "$TASK13_SERVER_WORKSPACE_CONFIG"
|
||||
|
||||
mkdir -p "$TASK13_SERVER_DATA" "$TASK13_SERVER_PI_STATE" "$TASK13_SERVER_REGISTRY"
|
||||
chmod 0777 "$TASK13_SERVER_DATA" "$TASK13_SERVER_PI_STATE" "$TASK13_SERVER_REGISTRY"
|
||||
data_root="$TASK13_SERVER_DATA"
|
||||
pi_root="$TASK13_SERVER_PI_STATE"
|
||||
registry_root="$TASK13_SERVER_REGISTRY"
|
||||
remote_path="$TASK13_REMOTE"
|
||||
workspace_path="$TASK13_SERVER_WORKSPACE_CONFIG"
|
||||
|
||||
cat >"$TASK13_OVERRIDE" <<EOF
|
||||
services:
|
||||
core:
|
||||
image: $TASK13_CORE_IMAGE
|
||||
build:
|
||||
labels:
|
||||
io.thothii.task13.run: "$TASK13_RUN_ID"
|
||||
environment:
|
||||
THT_WS_TASK13_SMOKE_DWH_HOST: dwh.task13.invalid
|
||||
THT_WS_TASK13_SMOKE_DWH_PORT: "5432"
|
||||
THT_WS_TASK13_SMOKE_DWH_USER: task13_reader
|
||||
THT_WS_TASK13_SMOKE_DWH_PASSWORD_FILE: /run/secrets/thothii.secrets
|
||||
THT_WS_TASK13_SMOKE_VECTOR_HOST: vector.task13.invalid
|
||||
THT_WS_TASK13_SMOKE_VECTOR_PORT: "5432"
|
||||
THT_WS_TASK13_SMOKE_VECTOR_USER: task13_vector_reader
|
||||
THT_WS_TASK13_SMOKE_VECTOR_PASSWORD_FILE: /run/secrets/thothii.secrets
|
||||
THT_WS_TASK13_SMOKE_EMBEDDING_BASE_URL: https://embedding.task13.invalid
|
||||
labels:
|
||||
io.thothii.task13.run: "$TASK13_RUN_ID"
|
||||
volumes:
|
||||
- $remote_path:/fixtures/remote.git:ro
|
||||
frontend:
|
||||
image: $TASK13_FRONTEND_IMAGE
|
||||
build:
|
||||
labels:
|
||||
io.thothii.task13.run: "$TASK13_RUN_ID"
|
||||
labels:
|
||||
io.thothii.task13.run: "$TASK13_RUN_ID"
|
||||
session-migrate:
|
||||
image: $TASK13_CORE_IMAGE
|
||||
networks:
|
||||
thothii:
|
||||
labels:
|
||||
io.thothii.task13.run: "$TASK13_RUN_ID"
|
||||
EOF
|
||||
chmod 0600 "$TASK13_OVERRIDE"
|
||||
|
||||
{
|
||||
printf 'THOTH_HTTP_PORT=0\n'
|
||||
printf 'THOTH_SERVER_BIND=127.0.0.1\n'
|
||||
printf 'MAX_PI_PROCESSES=2\n'
|
||||
printf 'PI_AUTH_FILE=%s\n' "$TASK13_PI_AUTH"
|
||||
printf 'THT_SECRETS_FILE=%s\n' "$TASK13_SECRETS"
|
||||
printf 'THT_WORKSPACE_GIT_REMOTE=/fixtures/remote.git\n'
|
||||
printf 'THT_WORKSPACE_GIT_BRANCH=%s\n' "$TASK13_BRANCH"
|
||||
printf 'THT_WORKSPACE_GIT_AUTHOR_NAME=Task 13 Server Smoke\n'
|
||||
printf 'THT_WORKSPACE_GIT_AUTHOR_EMAIL=task13-server@example.invalid\n'
|
||||
printf 'THT_DATA_ROOT=%s\n' "$data_root"
|
||||
printf 'THT_PI_STATE_ROOT=%s\n' "$pi_root"
|
||||
printf 'THT_WORKSPACE_REGISTRY_ROOT=%s\n' "$registry_root"
|
||||
printf 'THT_SERVER_WORKSPACE_CONFIG=%s\n' "$workspace_path"
|
||||
printf 'THT_SESSION_DB_HOST=task13-session.invalid\n'
|
||||
printf 'THT_SESSION_DB_PORT=5432\n'
|
||||
printf 'THT_SESSION_DB_NAME=task13\n'
|
||||
printf 'THT_SESSION_RUNTIME_USER=task13_runtime\n'
|
||||
printf 'THT_SESSION_MIGRATOR_USER=task13_migrator\n'
|
||||
printf 'THT_SESSION_DB_SSLMODE=verify-full\n'
|
||||
printf 'THT_SESSION_RUNTIME_PASSWORD_SOURCE=%s\n' "$TASK13_SESSION_RUNTIME_PASSWORD"
|
||||
printf 'THT_SESSION_MIGRATOR_PASSWORD_SOURCE=%s\n' "$TASK13_SESSION_MIGRATOR_PASSWORD_FILE"
|
||||
printf 'THT_SESSION_CA_SOURCE=%s\n' "$TASK13_SESSION_CA"
|
||||
printf 'THT_LLM_URL=https://llm.task13.invalid/v1\n'
|
||||
} >"$TASK13_ENV_FILE"
|
||||
chmod 0600 "$TASK13_ENV_FILE"
|
||||
}
|
||||
|
||||
task13_seed_registry() {
|
||||
mkdir -p "$TASK13_SEED/workspaces"
|
||||
task13_run_logged "initialize bare workspace registry" \
|
||||
@@ -377,6 +549,14 @@ task13_start_stack() {
|
||||
task13_log_failure "deterministic local LLM fixture readiness"
|
||||
}
|
||||
|
||||
task13_start_server_stack() {
|
||||
printf '== Build and start isolated Linux server profile ==\n'
|
||||
task13_assert_rendered_contract
|
||||
task13_compose_logged "build server Compose images" build --pull core frontend
|
||||
task13_compose_logged "start server Compose distribution" \
|
||||
up --detach --wait --wait-timeout 120 core frontend
|
||||
}
|
||||
|
||||
task13_frontend_address() {
|
||||
task13_compose port frontend 8080 | awk 'NR == 1 {print $0}'
|
||||
}
|
||||
@@ -422,6 +602,67 @@ task13_assert_runtime() {
|
||||
task13_run_logged "thothctl Pi doctor" "$TASK13_THOTHCTL" --installation "$TASK13_INSTALLATION" pi doctor
|
||||
}
|
||||
|
||||
task13_assert_server_runtime() {
|
||||
local frontend unauthenticated authenticated session_status core_id frontend_id
|
||||
local expected_core_image expected_frontend_image
|
||||
frontend="$(task13_frontend_address)"
|
||||
task13_run_logged "server frontend health" curl \
|
||||
--connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" --max-time "$TASK13_CURL_MAX_TIME" \
|
||||
--fail --silent --show-error "http://$frontend/"
|
||||
task13_run_logged "server same-origin core health" curl \
|
||||
--connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" --max-time "$TASK13_CURL_MAX_TIME" \
|
||||
--fail --silent --show-error "http://$frontend/api/health"
|
||||
|
||||
core_id="$(task13_core_id)"
|
||||
frontend_id="$(task13_compose ps -q frontend)"
|
||||
expected_core_image="$(docker image inspect --format '{{.Id}}' "$TASK13_CORE_IMAGE")"
|
||||
expected_frontend_image="$(docker image inspect --format '{{.Id}}' "$TASK13_FRONTEND_IMAGE")"
|
||||
[[ "$(docker inspect --format '{{.Image}}' "$core_id")" == "$expected_core_image" ]] \
|
||||
|| task13_fail "server core did not use the smoke-built core image"
|
||||
[[ "$(docker inspect --format '{{.Image}}' "$frontend_id")" == "$expected_frontend_image" ]] \
|
||||
|| task13_fail "server frontend did not use the smoke-built frontend image"
|
||||
task13_compose exec -T core sh -ceu '
|
||||
test "$AUTH_MODE" = upstream
|
||||
test "$THT_SESSION_STORAGE" = postgres
|
||||
test -r /run/secrets/thothii.secrets
|
||||
test -r /run/secrets/session_runtime_password
|
||||
test -r /run/secrets/session_ca.pem
|
||||
test -r /app/harness/workspaces/server-sessions.yaml
|
||||
'
|
||||
task13_mount_fingerprint | grep -Fq '/data = bind :' \
|
||||
|| task13_fail "server profile did not bind the disposable data root"
|
||||
task13_mount_fingerprint | grep -Fq '/home/thoth/.pi = bind :' \
|
||||
|| task13_fail "server profile did not bind the disposable Pi state root"
|
||||
task13_mount_fingerprint | grep -Fq '/data/workspace-registry = bind :' \
|
||||
|| task13_fail "server profile did not bind the disposable registry root"
|
||||
|
||||
unauthenticated="$(curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" \
|
||||
--max-time "$TASK13_CURL_MAX_TIME" --silent --output /dev/null --write-out '%{http_code}' \
|
||||
"http://$frontend/api/workspaces")"
|
||||
[[ "$unauthenticated" == 401 ]] || task13_fail "server profile did not enforce upstream auth"
|
||||
authenticated="$TASK13_TMP/server-workspaces.out"
|
||||
curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" --max-time "$TASK13_CURL_MAX_TIME" \
|
||||
--fail --silent --show-error \
|
||||
-H 'x-thoth-principal-issuer: task13-proxy' \
|
||||
-H 'x-thoth-principal-subject: task13-user' \
|
||||
-H 'x-thoth-principal-display-name: Task 13 User' \
|
||||
"http://$frontend/api/workspaces" >"$authenticated"
|
||||
grep -Fq 'Task 13 Smoke' "$authenticated" \
|
||||
|| task13_fail "authenticated server route did not expose the disposable registry"
|
||||
|
||||
session_status="$(curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" \
|
||||
--max-time "$TASK13_CURL_MAX_TIME" --silent --output "$TASK13_TMP/server-sessions.out" \
|
||||
--write-out '%{http_code}' \
|
||||
-H 'x-thoth-principal-issuer: task13-proxy' \
|
||||
-H 'x-thoth-principal-subject: task13-user' \
|
||||
"http://$frontend/api/sessions")"
|
||||
[[ "$session_status" == 503 ]] \
|
||||
|| task13_fail "disposable unavailable session dependency did not fail closed with 503"
|
||||
if grep -Fq "$TASK13_SECRET_VALUE" "$TASK13_TMP/server-sessions.out"; then
|
||||
task13_fail "server session failure exposed the fixture secret"
|
||||
fi
|
||||
}
|
||||
|
||||
task13_registry_status() {
|
||||
task13_compose exec -T core curl --connect-timeout 3 --max-time 10 -fsS \
|
||||
http://127.0.0.1:8787/workspace-registry/status
|
||||
@@ -523,10 +764,18 @@ task13_registry_lifecycle() {
|
||||
}
|
||||
|
||||
task13_prepare_bad_candidate() {
|
||||
task13_run_logged "pull pinned dead-core candidate" docker image pull "$TASK13_BAD_CANDIDATE_IMAGE"
|
||||
task13_run_logged "pull pinned stopped-core candidate" docker image pull "$TASK13_BAD_CANDIDATE_IMAGE"
|
||||
TASK13_BAD_CANDIDATE_ID="$(docker image inspect --format '{{.Id}}' "$TASK13_BAD_CANDIDATE_IMAGE")"
|
||||
[[ "$TASK13_BAD_CANDIDATE_ID" =~ ^sha256:[0-9a-f]{64}$ ]] \
|
||||
|| task13_fail "bad candidate image identity was not resolved"
|
||||
task13_run_logged "prove bad candidate exits" docker run \
|
||||
--name "$TASK13_BAD_CANDIDATE_CONTAINER" \
|
||||
--label "io.thothii.task13.run=$TASK13_RUN_ID" \
|
||||
"$TASK13_BAD_CANDIDATE_IMAGE"
|
||||
[[ "$(docker container inspect --format '{{.State.Running}}:{{.State.ExitCode}}' \
|
||||
"$TASK13_BAD_CANDIDATE_CONTAINER")" == false:0 ]] \
|
||||
|| task13_fail "bad candidate did not reach the guaranteed stopped state"
|
||||
task13_remove_labeled_container "$TASK13_BAD_CANDIDATE_CONTAINER"
|
||||
}
|
||||
|
||||
task13_update_rollback() {
|
||||
@@ -624,7 +873,12 @@ task13_remove_transaction_image() {
|
||||
task13_assert_project_ownership() {
|
||||
local kind id ids label
|
||||
for kind in container volume network; do
|
||||
if ! ids="$(docker "$kind" ls -q --filter "label=com.docker.compose.project=$TASK13_PROJECT")"; then
|
||||
if [[ "$kind" == container ]]; then
|
||||
if ! ids="$(docker container ls -aq --filter "label=com.docker.compose.project=$TASK13_PROJECT")"; then
|
||||
task13_fail "could not enumerate Compose project container resources"
|
||||
return 1
|
||||
fi
|
||||
elif ! ids="$(docker "$kind" ls -q --filter "label=com.docker.compose.project=$TASK13_PROJECT")"; then
|
||||
task13_fail "could not enumerate Compose project $kind resources"
|
||||
return 1
|
||||
fi
|
||||
@@ -674,6 +928,7 @@ task13_cleanup() {
|
||||
printf '%s\n' '--- sanitized Task 13 diagnostic log ---' >&2
|
||||
tail -n 200 "$TASK13_LOG" | task13_sanitize >&2
|
||||
fi
|
||||
task13_remove_labeled_container "${TASK13_BAD_CANDIDATE_CONTAINER:-}" || cleanup_rc=1
|
||||
task13_remove_labeled_container "${TASK13_LLM_CONTAINER:-}" || cleanup_rc=1
|
||||
if [[ -n "${TASK13_PROJECT:-}" && -n "${TASK13_ROOT:-}" && -f "${TASK13_OVERRIDE:-}" ]]; then
|
||||
if task13_assert_project_ownership >>"${TASK13_LOG:-/dev/null}" 2>&1; then
|
||||
@@ -899,6 +1154,177 @@ task13_self_test_transaction_image_cleanup() {
|
||||
rm -f "$calls" "$foreign_error"
|
||||
}
|
||||
|
||||
task13_self_test_rollback_fixture_contract() {
|
||||
[[ "${TASK13_BAD_CANDIDATE_BEHAVIOR:-}" == stopped ]] \
|
||||
|| task13_fail "rollback candidate is not declared as guaranteed stopped"
|
||||
[[ "$TASK13_BAD_CANDIDATE_IMAGE" =~ ^hello-world@sha256:[0-9a-f]{64}$ ]] \
|
||||
|| task13_fail "rollback candidate is not the digest-pinned stopped fixture"
|
||||
}
|
||||
|
||||
task13_self_test_runtime_binding_fixture() {
|
||||
local fixture_source
|
||||
fixture_source="$(declare -f task13_write_fixture_files)"
|
||||
for variable in \
|
||||
THT_WS_TASK13_SMOKE_DWH_HOST \
|
||||
THT_WS_TASK13_SMOKE_DWH_PORT \
|
||||
THT_WS_TASK13_SMOKE_DWH_USER \
|
||||
THT_WS_TASK13_SMOKE_DWH_PASSWORD_FILE \
|
||||
THT_WS_TASK13_SMOKE_VECTOR_HOST \
|
||||
THT_WS_TASK13_SMOKE_VECTOR_PORT \
|
||||
THT_WS_TASK13_SMOKE_VECTOR_USER \
|
||||
THT_WS_TASK13_SMOKE_VECTOR_PASSWORD_FILE \
|
||||
THT_WS_TASK13_SMOKE_EMBEDDING_BASE_URL; do
|
||||
grep -Fq "$variable" <<<"$fixture_source" \
|
||||
|| task13_fail "rollback fixture lacks runtime binding: $variable"
|
||||
done
|
||||
}
|
||||
|
||||
task13_self_test_server_runtime_binding_fixture() {
|
||||
local fixture_source
|
||||
fixture_source="$(declare -f task13_write_server_fixture_files)"
|
||||
for variable in \
|
||||
THT_WS_TASK13_SMOKE_DWH_HOST \
|
||||
THT_WS_TASK13_SMOKE_DWH_PASSWORD_FILE \
|
||||
THT_WS_TASK13_SMOKE_VECTOR_HOST \
|
||||
THT_WS_TASK13_SMOKE_VECTOR_PASSWORD_FILE \
|
||||
THT_WS_TASK13_SMOKE_EMBEDDING_BASE_URL; do
|
||||
grep -Fq "$variable" <<<"$fixture_source" \
|
||||
|| task13_fail "server fixture lacks runtime binding: $variable"
|
||||
done
|
||||
}
|
||||
|
||||
task13_self_test_stopped_project_containers() {
|
||||
local calls foreign_error
|
||||
calls="$(mktemp "${TMPDIR:-/tmp}/thothii-task13-project-containers.XXXXXX")"
|
||||
foreign_error="$calls.foreign-error"
|
||||
TASK13_PROJECT="thothii-0123456789ab"
|
||||
TASK13_RUN_ID="task13-contract-run"
|
||||
|
||||
docker() {
|
||||
printf '%s\n' "$*" >>"$calls"
|
||||
case "$1 $2 $3" in
|
||||
"container ls -aq") printf '%s\n' stopped-foreign ;;
|
||||
"container inspect --format") printf '%s\n' some-other-run ;;
|
||||
"volume ls -q"|"network ls -q") : ;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
if task13_assert_project_ownership 2>"$foreign_error"; then
|
||||
unset -f docker
|
||||
rm -f "$calls" "$foreign_error"
|
||||
task13_fail "project cleanup accepted a stopped foreign container"
|
||||
fi
|
||||
grep -Fq 'container ls -aq' "$calls" \
|
||||
|| task13_fail "project cleanup did not enumerate stopped containers"
|
||||
grep -Fq 'Compose project contains a foreign container resource' "$foreign_error" \
|
||||
|| task13_fail "stopped foreign container refusal was not explicit"
|
||||
|
||||
: >"$calls"
|
||||
docker() {
|
||||
printf '%s\n' "$*" >>"$calls"
|
||||
case "$1 $2 $3" in
|
||||
"container ls -aq") printf '%s\n' stopped-owned ;;
|
||||
"container inspect --format") printf '%s\n' "$TASK13_RUN_ID" ;;
|
||||
"volume ls -q"|"network ls -q") : ;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
}
|
||||
task13_assert_project_ownership
|
||||
[[ "$(grep -Fc 'container inspect --format' "$calls")" -eq 1 ]] \
|
||||
|| task13_fail "project cleanup did not inspect exactly the stopped owned container"
|
||||
unset -f docker
|
||||
rm -f "$calls" "$foreign_error"
|
||||
}
|
||||
|
||||
task13_self_test_timeout_process_group() {
|
||||
local child_file child_pid="" rc
|
||||
child_file="$(mktemp "${TMPDIR:-/tmp}/thothii-task13-timeout-child.XXXXXX")"
|
||||
set +e
|
||||
task13_bounded 1 "child-process regression" bash -c \
|
||||
'sleep 30 & printf "%s\n" "$!" >"$1"; wait' _ "$child_file" >/dev/null 2>&1
|
||||
rc=$?
|
||||
set -e
|
||||
child_pid="$(sed -n '1p' "$child_file")"
|
||||
[[ "$rc" -ne 0 ]] || {
|
||||
rm -f "$child_file"
|
||||
task13_fail "timed command unexpectedly succeeded"
|
||||
}
|
||||
if [[ -n "$child_pid" ]] && kill -0 "$child_pid" 2>/dev/null; then
|
||||
kill -KILL "$child_pid" 2>/dev/null || true
|
||||
rm -f "$child_file"
|
||||
task13_fail "timed command left its child process alive"
|
||||
fi
|
||||
rm -f "$child_file"
|
||||
}
|
||||
|
||||
task13_self_test_nested_timeout_process_group() {
|
||||
local child_file child_pid="" rc
|
||||
child_file="$(mktemp "${TMPDIR:-/tmp}/thothii-task13-nested-timeout.XXXXXX")"
|
||||
task13_nested_timeout_fixture() {
|
||||
task13_bounded 30 "nested child-process regression" bash -c \
|
||||
'sleep 30 & printf "%s\n" "$!" >"$1"; wait' _ "$child_file"
|
||||
}
|
||||
set +e
|
||||
task13_supervise 1 "nested timeout supervisor" task13_nested_timeout_fixture >/dev/null 2>&1
|
||||
rc=$?
|
||||
set -e
|
||||
unset -f task13_nested_timeout_fixture
|
||||
child_pid="$(sed -n '1p' "$child_file")"
|
||||
[[ "$rc" -ne 0 ]] || {
|
||||
rm -f "$child_file"
|
||||
task13_fail "nested timed command unexpectedly succeeded"
|
||||
}
|
||||
if [[ -n "$child_pid" ]] && kill -0 "$child_pid" 2>/dev/null; then
|
||||
kill -KILL "$child_pid" 2>/dev/null || true
|
||||
rm -f "$child_file"
|
||||
task13_fail "outer timeout left its nested command child alive"
|
||||
fi
|
||||
rm -f "$child_file"
|
||||
}
|
||||
|
||||
task13_self_test_public_timeout_contract() {
|
||||
local root
|
||||
root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)"
|
||||
grep -Eq 'task13_supervise[[:space:]].*task13_smoke_main[[:space:]]+full' \
|
||||
"$root/scripts/unified-deployment-smoke.sh" \
|
||||
|| task13_fail "direct unified smoke invocation lacks an internal supervisor"
|
||||
grep -Eq 'task13_supervise[[:space:]].*task13_smoke_main[[:space:]]+update' \
|
||||
"$root/scripts/thothctl-update-smoke.sh" \
|
||||
|| task13_fail "direct update smoke invocation lacks an internal supervisor"
|
||||
}
|
||||
|
||||
task13_self_test_windows_release_contract() {
|
||||
local root script workflow
|
||||
root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)"
|
||||
script="$root/scripts/test-windows-clone-contract.ps1"
|
||||
workflow="$root/.github/workflows/deployment.yml"
|
||||
grep -Fq 'Task 13 path with spaces' "$script" \
|
||||
|| task13_fail "Windows contract does not operate from a path containing spaces"
|
||||
grep -Fq 'DockerStartup' "$script" \
|
||||
|| task13_fail "Windows contract lacks an explicit Docker startup mode"
|
||||
grep -Fq 'Kill($true)' "$script" \
|
||||
|| task13_fail "Windows bounded runner does not kill the full process tree"
|
||||
grep -Fq 'docker-desktop' "$workflow" \
|
||||
|| task13_fail "workflow lacks a manual self-hosted Windows Docker Desktop gate"
|
||||
grep -Fq -- '-DockerStartup' "$workflow" \
|
||||
|| task13_fail "manual Windows release job does not execute Docker startup mode"
|
||||
}
|
||||
|
||||
task13_self_test_server_release_contract() {
|
||||
local root workflow server_smoke
|
||||
root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)"
|
||||
workflow="$root/.github/workflows/deployment.yml"
|
||||
server_smoke="$root/scripts/server-deployment-smoke.sh"
|
||||
[[ -x "$server_smoke" ]] || task13_fail "bounded Linux server deployment smoke is missing"
|
||||
grep -Fq 'deploy/compose.server.yaml' "$root/scripts/unified-deployment-smoke.sh" \
|
||||
|| task13_fail "server smoke does not load the server profile"
|
||||
grep -Fq 'deploy/compose.session-server.yaml.example' "$root/scripts/unified-deployment-smoke.sh" \
|
||||
|| task13_fail "server smoke does not load the required session overlay"
|
||||
grep -Eq 'timeout .*scripts/server-deployment-smoke\.sh' "$workflow" \
|
||||
|| task13_fail "workflow lacks an outer timeout for the Linux server smoke"
|
||||
}
|
||||
|
||||
task13_self_test_source_contract() {
|
||||
local root host_network push_command registry_function workflow uses_count pinned_uses_count
|
||||
root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)"
|
||||
@@ -950,10 +1376,34 @@ task13_self_test() {
|
||||
task13_self_test_cleanup_ownership
|
||||
task13_self_test_image_cleanup_ownership
|
||||
task13_self_test_transaction_image_cleanup
|
||||
task13_self_test_rollback_fixture_contract
|
||||
task13_self_test_runtime_binding_fixture
|
||||
task13_self_test_server_runtime_binding_fixture
|
||||
task13_self_test_stopped_project_containers
|
||||
task13_self_test_timeout_process_group
|
||||
task13_self_test_nested_timeout_process_group
|
||||
task13_self_test_public_timeout_contract
|
||||
task13_self_test_windows_release_contract
|
||||
task13_self_test_server_release_contract
|
||||
task13_self_test_source_contract
|
||||
printf 'Task 13 smoke safety contracts passed.\n'
|
||||
}
|
||||
|
||||
task13_self_test_case() {
|
||||
case "$1" in
|
||||
rollback) task13_self_test_rollback_fixture_contract ;;
|
||||
runtime-bindings) task13_self_test_runtime_binding_fixture ;;
|
||||
server-bindings) task13_self_test_server_runtime_binding_fixture ;;
|
||||
cleanup) task13_self_test_stopped_project_containers ;;
|
||||
timeout-group) task13_self_test_timeout_process_group ;;
|
||||
timeout-nested) task13_self_test_nested_timeout_process_group ;;
|
||||
timeout-public) task13_self_test_public_timeout_contract ;;
|
||||
windows) task13_self_test_windows_release_contract ;;
|
||||
server) task13_self_test_server_release_contract ;;
|
||||
*) task13_fail "unknown Task 13 self-test case: $1" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
task13_initialize() {
|
||||
umask 077
|
||||
TASK13_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)"
|
||||
@@ -966,6 +1416,7 @@ task13_initialize() {
|
||||
trap 'task13_cleanup $?' EXIT
|
||||
trap 'exit 130' INT TERM HUP
|
||||
TASK13_RUN_ID="$(date -u +%Y%m%d%H%M%S)-$$-${RANDOM:-0}"
|
||||
TASK13_PROFILE="local"
|
||||
TASK13_INSTALLATION="$TASK13_TMP/thothii-installation.yaml"
|
||||
TASK13_PROJECT="thothii-$(task13_sha256_text "$TASK13_INSTALLATION" | cut -c1-12)"
|
||||
TASK13_CONTROL_DIR="$TASK13_ROOT/.thothctl/$TASK13_PROJECT"
|
||||
@@ -984,12 +1435,22 @@ task13_initialize() {
|
||||
TASK13_LLM_SERVER="$TASK13_TMP/fake-llm.mjs"
|
||||
TASK13_THOTHCTL_DIR="$TASK13_TMP/thothctl"
|
||||
TASK13_LLM_CONTAINER="$TASK13_PROJECT-llm"
|
||||
TASK13_BAD_CANDIDATE_CONTAINER="$TASK13_PROJECT-bad-candidate"
|
||||
TASK13_CORE_IMAGE="task13-core-$TASK13_RUN_ID:local"
|
||||
TASK13_FRONTEND_IMAGE="task13-frontend-$TASK13_RUN_ID:local"
|
||||
TASK13_SECRET_VALUE="task13-secret-$TASK13_RUN_ID"
|
||||
TASK13_NETWORK=""
|
||||
TASK13_BAD_CANDIDATE_ID=""
|
||||
TASK13_PREVIOUS_IMAGE_ID=""
|
||||
TASK13_SERVER_DATA="$TASK13_TMP/Server Data"
|
||||
TASK13_SERVER_PI_STATE="$TASK13_TMP/Server Pi State"
|
||||
TASK13_SERVER_REGISTRY="$TASK13_TMP/Server Registry"
|
||||
TASK13_SERVER_WORKSPACE_CONFIG="$TASK13_TMP/server-sessions.yaml"
|
||||
TASK13_SESSION_RUNTIME_PASSWORD="$TASK13_TMP/session-runtime-password"
|
||||
TASK13_SESSION_MIGRATOR_PASSWORD_FILE="$TASK13_TMP/session-migrator-password"
|
||||
TASK13_SESSION_CA="$TASK13_TMP/session-ca.pem"
|
||||
TASK13_SESSION_PASSWORD="task13-runtime-$TASK13_RUN_ID"
|
||||
TASK13_SESSION_MIGRATOR_PASSWORD="task13-migrator-$TASK13_RUN_ID"
|
||||
}
|
||||
|
||||
task13_require_tools() {
|
||||
@@ -1022,10 +1483,26 @@ task13_smoke_main() {
|
||||
printf 'Task 13 %s deployment smoke passed.\n' "$mode"
|
||||
}
|
||||
|
||||
task13_server_smoke_main() {
|
||||
task13_initialize
|
||||
TASK13_PROFILE="server"
|
||||
task13_require_tools
|
||||
task13_write_server_fixture_files
|
||||
task13_seed_registry
|
||||
task13_start_server_stack
|
||||
task13_assert_project_ownership
|
||||
task13_assert_built_image_ownership
|
||||
task13_assert_server_runtime
|
||||
printf 'Task 13 Linux server deployment smoke passed.\n'
|
||||
}
|
||||
|
||||
if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
|
||||
if [[ "${1:-}" == "--self-test" ]]; then
|
||||
task13_self_test
|
||||
elif [[ "${1:-}" == "--self-test-case" ]]; then
|
||||
[[ -n "${2:-}" ]] || task13_fail "--self-test-case requires a case name"
|
||||
task13_self_test_case "$2"
|
||||
else
|
||||
task13_smoke_main full
|
||||
task13_supervise "$TASK13_SMOKE_TIMEOUT" "unified deployment smoke" task13_smoke_main full
|
||||
fi
|
||||
fi
|
||||
|
||||
Reference in New Issue
Block a user