mptyl/ThothII
1
0
Fork 0
Code Issues 25 Pull Requests Actions Packages Projects Releases 1 Wiki Activity
Files
2ae89c075ea2ab583fd3b7009e3e0d22c8ba5769
ThothII/scripts/unified-deployment-smoke.sh
T

1032 lines
40 KiB
Bash
Executable File
Raw Blame History

#!/usr/bin/env bash
# End-to-end release gate for the canonical two-service Compose distribution.
# This file is also sourced by thothctl-update-smoke.sh so both entry points use the same
# isolated fixture, exact cleanup, and sanitized failure reporting.
set -euo pipefail
TASK13_BAD_CANDIDATE_IMAGE="registry:2.8.3@sha256:a3d8aaa63ed8681a604f1dea0aa03f100d5895b6a58ace528858a7b332415373"
TASK13_BAD_PI_VERSION="0.80.4-task13"
TASK13_CURL_CONNECT_TIMEOUT=3
TASK13_CURL_MAX_TIME=10
TASK13_CLEANUP_TIMEOUT=20
task13_fail() {
printf 'Task 13 smoke failed: %s\n' "$*" >&2
return 1
}
task13_sha256_text() {
if command -v sha256sum >/dev/null 2>&1; then
printf '%s' "$1" | sha256sum | awk '{print $1}'
elif command -v shasum >/dev/null 2>&1; then
printf '%s' "$1" | shasum -a 256 | awk '{print $1}'
else
task13_fail "sha256sum or shasum is required"
fi
}
task13_sanitize() {
local line
while IFS= read -r line || [[ -n "$line" ]]; do
if [[ -n "${TASK13_SECRET_VALUE:-}" ]]; then
line="${line//"$TASK13_SECRET_VALUE"/[REDACTED]}"
fi
printf '%s\n' "$line"
done | sed -E \
-e 's#([[:alpha:]][[:alnum:]+.-]*://[^:/@[:space:]]+:)[^@/[:space:]]+@#\1[REDACTED]@#g' \
-e 's/(([Pp]roxy-)?[Aa]uthorization:[[:space:]]*([Bb]earer|[Bb]asic)[[:space:]]+)[^[:space:]]+/\1[REDACTED]/g' \
-e "s/(([\"']?[[:alnum:]_.-]*(password|token|api[_-]?key|secret|key)[[:alnum:]_.-]*[\"']?[[:space:]]*[:=][[:space:]]*)([\"'][^\"']*[\"']|[^[:space:],;]+))/\2[REDACTED]/Ig"
}
task13_log_failure() {
local label="$1"
TASK13_FAILURE_LOGGED=1
printf 'Task 13 command failed: %s\n' "$label" >&2
tail -n 200 "$TASK13_LOG" | task13_sanitize >&2
return 1
}
task13_run_logged() {
local label="$1"
shift
if ! "$@" >>"$TASK13_LOG" 2>&1; then
task13_log_failure "$label"
fi
}
task13_bounded() {
local seconds="$1" label="$2" command_pid watchdog_pid rc
shift 2
"$@" &
command_pid=$!
(
local sleep_pid
sleep "$seconds" &
sleep_pid=$!
trap 'kill "$sleep_pid" 2>/dev/null || true' EXIT INT TERM
wait "$sleep_pid" 2>/dev/null || exit 0
trap - EXIT INT TERM
if kill -0 "$command_pid" 2>/dev/null; then
printf 'Task 13 command timed out after %ss: %s\n' "$seconds" "$label" >&2
kill -TERM "$command_pid" 2>/dev/null || true
sleep 5
kill -KILL "$command_pid" 2>/dev/null || true
fi
) &
watchdog_pid=$!
if wait "$command_pid"; then
rc=0
else
rc=$?
fi
kill "$watchdog_pid" 2>/dev/null || true
wait "$watchdog_pid" 2>/dev/null || true
return "$rc"
}
task13_compose_files() {
TASK13_COMPOSE=(
docker compose
--project-name "$TASK13_PROJECT"
--project-directory "$TASK13_ROOT"
--env-file "$TASK13_ENV_FILE"
-f "$TASK13_ROOT/compose.yaml"
-f "$TASK13_ROOT/deploy/compose.local.yaml"
-f "$TASK13_OVERRIDE"
)
if [[ -f "$TASK13_CURRENT_IMAGE_OVERRIDE" ]]; then
TASK13_COMPOSE+=(-f "$TASK13_CURRENT_IMAGE_OVERRIDE")
fi
}
task13_compose() {
task13_compose_files
"${TASK13_COMPOSE[@]}" "$@"
}
task13_compose_logged() {
local label="$1"
shift
task13_compose_files
task13_run_logged "$label" "${TASK13_COMPOSE[@]}" "$@"
}
task13_write_environment() {
local remote="$1"
{
printf 'THOTH_HTTP_PORT=0\n'
printf 'THOTH_CORE_HTTP_PORT=0\n'
printf 'MAX_PI_PROCESSES=2\n'
printf 'PI_AUTH_FILE=%s\n' "$TASK13_PI_AUTH"
printf 'THT_SECRETS_FILE=%s\n' "$TASK13_SECRETS"
printf 'THT_WORKSPACE_GIT_REMOTE=%s\n' "$remote"
printf 'THT_WORKSPACE_GIT_BRANCH=%s\n' "$TASK13_BRANCH"
printf 'THT_WORKSPACE_GIT_AUTHOR_NAME=Task 13 Smoke\n'
printf 'THT_WORKSPACE_GIT_AUTHOR_EMAIL=task13-smoke@example.invalid\n'
printf 'THT_LLM_URL=http://%s:9000/v1\n' "$TASK13_LLM_CONTAINER"
} >"$TASK13_ENV_FILE"
chmod 0600 "$TASK13_ENV_FILE"
}
task13_write_fixture_files() {
printf '{}\n' >"$TASK13_PI_AUTH"
printf 'THT_MODEL_API_KEY=%s\n' "$TASK13_SECRET_VALUE" >"$TASK13_SECRETS"
chmod 0644 "$TASK13_PI_AUTH"
chmod 0600 "$TASK13_SECRETS"
cat >"$TASK13_PI_MODELS" <<EOF
{
"providers": {
"local-qwen": {
"baseUrl": "http://$TASK13_LLM_CONTAINER:9000/v1",
"api": "openai-completions",
"apiKey": "task13-local-provider",
"models": [
{
"id": "task13-smoke",
"name": "Task 13 deterministic smoke",
"reasoning": false,
"contextWindow": 4096,
"maxTokens": 64
}
]
}
}
}
EOF
cat >"$TASK13_PI_SETTINGS" <<'EOF'
{
"defaultProjectTrust": "always",
"enabledModels": ["local-qwen/task13-smoke"]
}
EOF
chmod 0644 "$TASK13_PI_MODELS" "$TASK13_PI_SETTINGS"
cat >"$TASK13_LLM_SERVER" <<'EOF'
import http from "node:http";
const server = http.createServer((request, response) => {
if (request.method === "GET" && request.url === "/health") {
response.writeHead(200, { "content-type": "application/json" });
response.end('{"status":"ok"}');
return;
}
if (request.method === "GET" && request.url === "/v1/models") {
response.writeHead(200, { "content-type": "application/json" });
response.end('{"object":"list","data":[{"id":"task13-smoke","object":"model"}]}');
return;
}
if (request.method !== "POST" || request.url !== "/v1/chat/completions") {
response.writeHead(404, { "content-type": "application/json" });
response.end('{"error":{"message":"not found"}}');
return;
}
let body = "";
request.setEncoding("utf8");
request.on("data", (chunk) => { body += chunk; });
request.on("end", () => {
let stream = true;
try { stream = JSON.parse(body).stream !== false; } catch { /* return the safe fixture */ }
if (!stream) {
response.writeHead(200, { "content-type": "application/json" });
response.end(JSON.stringify({
id: "task13", object: "chat.completion", created: 1, model: "task13-smoke",
choices: [{ index: 0, message: { role: "assistant", content: "OK" }, finish_reason: "stop" }],
}));
return;
}
response.writeHead(200, {
"content-type": "text/event-stream",
"cache-control": "no-cache",
connection: "keep-alive",
});
response.write('data: {"id":"task13","object":"chat.completion.chunk","created":1,"model":"task13-smoke","choices":[{"index":0,"delta":{"role":"assistant","content":"OK"},"finish_reason":null}]}\n\n');
response.write('data: {"id":"task13","object":"chat.completion.chunk","created":1,"model":"task13-smoke","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}\n\n');
response.end("data: [DONE]\n\n");
});
});
server.listen(9000, "0.0.0.0");
EOF
chmod 0644 "$TASK13_LLM_SERVER"
cat >"$TASK13_OVERRIDE" <<EOF
services:
core:
image: $TASK13_CORE_IMAGE
build:
labels:
io.thothii.task13.run: "$TASK13_RUN_ID"
environment:
PI_PROVIDER: local-qwen
PI_MODEL: task13-smoke
PI_THINKING: low
THT_WORKSPACE_INSTALLATION_ID: task13-smoke
THT_WORKSPACE_REGISTRY_ROOT: /data/workspace-registry
labels:
io.thothii.task13.run: "$TASK13_RUN_ID"
volumes: !override
- settings:/data/settings
- pi-state:/home/thoth/.pi
- $TASK13_PI_AUTH:/home/thoth/.pi/agent/auth.json:ro
- $TASK13_PI_MODELS:/home/thoth/.pi/agent/models.json:ro
- $TASK13_PI_SETTINGS:/home/thoth/.pi/agent/settings.json:ro
- workspace-registry:/data/workspace-registry
- sessions:/data/sessions
- $TASK13_REMOTE:/fixtures/remote.git:ro
frontend:
image: $TASK13_FRONTEND_IMAGE
build:
labels:
io.thothii.task13.run: "$TASK13_RUN_ID"
labels:
io.thothii.task13.run: "$TASK13_RUN_ID"
networks:
thothii:
labels:
io.thothii.task13.run: "$TASK13_RUN_ID"
volumes:
settings:
labels:
io.thothii.task13.run: "$TASK13_RUN_ID"
pi-state:
labels:
io.thothii.task13.run: "$TASK13_RUN_ID"
workspace-registry:
labels:
io.thothii.task13.run: "$TASK13_RUN_ID"
sessions:
labels:
io.thothii.task13.run: "$TASK13_RUN_ID"
EOF
chmod 0600 "$TASK13_OVERRIDE"
cat >"$TASK13_INSTALLATION" <<EOF
profile: local
projectDirectory: "$TASK13_ROOT"
envFile: "$TASK13_ENV_FILE"
overrides:
- "$TASK13_OVERRIDE"
EOF
chmod 0600 "$TASK13_INSTALLATION"
}
task13_seed_registry() {
mkdir -p "$TASK13_SEED/workspaces"
task13_run_logged "initialize bare workspace registry" \
git init --bare --initial-branch=main "$TASK13_REMOTE"
task13_run_logged "initialize workspace seed" git -C "$TASK13_SEED" init --initial-branch=main
cat >"$TASK13_SEED/workspaces/task13-smoke.yaml" <<'EOF'
workspace:
schema_version: 2
id: task13-smoke
name: Task 13 Smoke
language: en
dwh:
engine: postgres
database: warehouse
schema: analytics
supported_transports: [postgres_direct]
semantic_index:
vector_store:
engine: pgvector
database: vectors
schema: public
collection: task13_documents
dimensions: 8
distance: cosine
supported_transports: [pgvector_direct]
embedding:
provider: ollama_compatible
model: task13-embedding
dimensions: 8
llm_policy:
default: local-qwen/task13-smoke
allowed: [local-qwen/task13-smoke]
EOF
task13_run_logged "commit initial workspace" git -C "$TASK13_SEED" add workspaces/task13-smoke.yaml
task13_run_logged "commit initial workspace" git -C "$TASK13_SEED" \
-c user.name='Task 13 Smoke' -c user.email='task13-smoke@example.invalid' \
commit -m 'Seed Task 13 workspace'
task13_run_logged "push initial workspace" git -C "$TASK13_SEED" \
push "$TASK13_REMOTE" "HEAD:$TASK13_BRANCH"
chmod -R a+rX "$TASK13_REMOTE"
}
task13_build_thothctl() {
local os arch
mkdir -p "$TASK13_THOTHCTL_DIR"
task13_run_logged "build thothctl cross-platform binaries" env \
THT_THOTHCTL_OUTPUT_DIRECTORY="$TASK13_THOTHCTL_DIR" \
bash "$TASK13_ROOT/scripts/build-thothctl.sh"
os="$(uname -s)"
arch="$(uname -m)"
case "$os/$arch" in
Darwin/x86_64) TASK13_THOTHCTL="$TASK13_THOTHCTL_DIR/thothctl-darwin-amd64" ;;
Darwin/arm64) TASK13_THOTHCTL="$TASK13_THOTHCTL_DIR/thothctl-darwin-arm64" ;;
Linux/x86_64|Linux/amd64) TASK13_THOTHCTL="$TASK13_THOTHCTL_DIR/thothctl-linux-amd64" ;;
Linux/aarch64|Linux/arm64) TASK13_THOTHCTL="$TASK13_THOTHCTL_DIR/thothctl-linux-arm64" ;;
*) task13_fail "unsupported smoke host: $os/$arch" ;;
esac
chmod 0700 "$TASK13_THOTHCTL"
task13_run_logged "invoke host thothctl" "$TASK13_THOTHCTL" --help
}
task13_assert_rendered_contract() {
local services rendered
services="$(task13_compose config --services | sort)"
[[ "$services" == $'core\nfrontend' ]] || task13_fail "rendered stack is not exactly core and frontend"
rendered="$TASK13_TMP/rendered-compose.yaml"
task13_compose config >"$rendered"
if grep -Eqi 'docker\.sock|/var/run/docker' "$rendered"; then
task13_fail "rendered Compose exposes a Docker daemon endpoint"
fi
if grep -Fq "$TASK13_SECRET_VALUE" "$rendered"; then
task13_fail "rendered Compose exposed the fixture secret"
fi
for endpoint in THT_DWH_REST_URL THT_VEC_REST_URL THT_OLLAMA_URL THT_LLM_URL; do
grep -Fq "$endpoint" "$rendered" || task13_fail "rendered Compose lacks $endpoint"
done
}
task13_start_stack() {
printf '== Build and start isolated local Compose distribution ==\n'
task13_assert_rendered_contract
task13_compose_logged "build local Compose images" build --pull
task13_compose_logged "start local Compose distribution" up --detach --wait --wait-timeout 120
TASK13_NETWORK="$(docker network ls \
--filter "label=com.docker.compose.project=$TASK13_PROJECT" \
--filter 'label=com.docker.compose.network=thothii' --format '{{.Name}}')"
[[ -n "$TASK13_NETWORK" && "$TASK13_NETWORK" != *$'\n'* ]] || task13_fail "isolated Compose network was not resolved"
task13_run_logged "start deterministic local LLM fixture" docker run --detach \
--name "$TASK13_LLM_CONTAINER" \
--label "io.thothii.task13.run=$TASK13_RUN_ID" \
--network "$TASK13_NETWORK" \
--entrypoint node \
--volume "$TASK13_LLM_SERVER:/fixtures/fake-llm.mjs:ro" \
"$TASK13_CORE_IMAGE" /fixtures/fake-llm.mjs
for _attempt in $(seq 1 30); do
if docker exec "$TASK13_LLM_CONTAINER" node -e \
"fetch('http://127.0.0.1:9000/health',{signal:AbortSignal.timeout(3000)}).then(r=>{if(!r.ok)process.exit(1)}).catch(()=>process.exit(1))" \
>>"$TASK13_LOG" 2>&1; then
return 0
fi
sleep 1
done
task13_log_failure "deterministic local LLM fixture readiness"
}
task13_frontend_address() {
task13_compose port frontend 8080 | awk 'NR == 1 {print $0}'
}
task13_core_id() {
task13_compose ps -q core
}
task13_assert_runtime() {
local frontend expected_pi actual_pi core_id
frontend="$(task13_frontend_address)"
expected_pi="$(sed -n 's/^ARG PI_VERSION=//p' "$TASK13_ROOT/docker/core.Dockerfile" | head -n 1)"
actual_pi="$(task13_compose exec -T core pi --version | tr -d '\r\n')"
[[ -n "$expected_pi" && "$actual_pi" == "$expected_pi" ]] || task13_fail "embedded Pi version mismatch"
task13_run_logged "frontend health" curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" \
--max-time "$TASK13_CURL_MAX_TIME" --fail --silent --show-error "http://$frontend/"
task13_run_logged "same-origin core health" curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" \
--max-time "$TASK13_CURL_MAX_TIME" --fail --silent --show-error "http://$frontend/api/health"
task13_compose_logged "core non-root identity" exec -T core sh -ceu \
'test "$(id -u)" = 10001'
task13_compose_logged "embedded Pi executable" exec -T core sh -ceu \
'command -v pi >/dev/null'
task13_compose_logged "core Docker socket isolation" exec -T core sh -ceu \
'test ! -e /var/run/docker.sock'
task13_compose_logged "workspace registry bootstrap" exec -T core \
curl --connect-timeout 3 --max-time 10 -fsS http://127.0.0.1:8787/workspace-registry/status
task13_compose_logged "active workspace registry state" exec -T core sh -ceu \
'test -f /data/workspace-registry/state/active.json'
task13_compose_logged "mounted Pi auth readability" exec -T core sh -ceu \
'test -r /home/thoth/.pi/agent/auth.json'
task13_compose_logged "mounted application secret readability" exec -T core sh -ceu \
'test -r /run/secrets/thothii.secrets'
core_id="$(task13_core_id)"
[[ "$(docker inspect --format '{{ index .Config.Labels "io.thothii.task13.run" }}' "$core_id")" == "$TASK13_RUN_ID" ]] \
|| task13_fail "core lacks the explicit Task 13 resource label"
task13_compose_logged "internal Pi provider smoke" exec -T core \
curl --connect-timeout 3 --max-time 45 -fsS -X POST \
-H 'x-thoth-principal-issuer: thothctl' \
-H 'x-thoth-principal-subject: thothctl-maintenance' \
-H 'x-thoth-principal-display-name: Thothctl maintenance' \
-H 'x-thoth-is-admin: 1' \
http://127.0.0.1:8787/pi-management/test
task13_run_logged "thothctl Pi doctor" "$TASK13_THOTHCTL" --installation "$TASK13_INSTALLATION" pi doctor
}
task13_registry_status() {
task13_compose exec -T core curl --connect-timeout 3 --max-time 10 -fsS \
http://127.0.0.1:8787/workspace-registry/status
}
task13_registry_head() {
sed -n 's/.*"head":"\([0-9a-f][0-9a-f]*\)".*/\1/p'
}
task13_active_registry_head() {
task13_compose exec -T core sed -n \
's/.*"head":"\([0-9a-f][0-9a-f]*\)".*/\1/p' \
/data/workspace-registry/state/active.json
}
task13_assert_sentinels() {
task13_compose exec -T core sh -ceu '
test "$(cat /data/settings/task13-settings)" = settings-preserved
test "$(cat /data/sessions/task13-session)" = sessions-preserved
test "$(cat /home/thoth/.pi/task13-pi-state)" = pi-state-preserved
test -f /data/workspace-registry/state/active.json
'
}
task13_mount_fingerprint() {
docker inspect --format '{{range .Mounts}}{{println .Destination "=" .Type ":" .Name}}{{end}}' "$(task13_core_id)" \
| LC_ALL=C sort
}
task13_prepare_persistence() {
task13_compose exec -T core sh -ceu '
printf %s settings-preserved > /data/settings/task13-settings
printf %s sessions-preserved > /data/sessions/task13-session
printf %s pi-state-preserved > /home/thoth/.pi/task13-pi-state
'
TASK13_INITIAL_MOUNTS="$(task13_mount_fingerprint)"
TASK13_INITIAL_HEAD="$(task13_active_registry_head)"
[[ "$TASK13_INITIAL_HEAD" =~ ^[0-9a-f]{40}$ ]] || task13_fail "initial registry head is invalid"
task13_compose exec -T core curl --connect-timeout 3 --max-time 10 -fsS \
http://127.0.0.1:8787/workspaces \
| grep -Fq 'Task 13 Smoke' || task13_fail "initial workspace is unavailable"
}
task13_registry_lifecycle() {
local offline_status offline_head valid_head
printf '== Recreate offline and retain the validated registry snapshot ==\n'
task13_write_environment /fixtures/offline.git
task13_compose_logged "offline Compose recreation" up --detach --force-recreate --wait --wait-timeout 120
offline_status="$(task13_registry_status)"
offline_head="$(printf '%s' "$offline_status" | task13_registry_head)"
[[ "$offline_head" == "$TASK13_INITIAL_HEAD" ]] || task13_fail "offline recreation changed registry head"
grep -Fq '"degraded":true' <<<"$offline_status" || task13_fail "offline recreation did not report degraded mode"
task13_assert_sentinels
[[ "$(task13_mount_fingerprint)" == "$TASK13_INITIAL_MOUNTS" ]] || task13_fail "offline recreation changed volume identity"
printf '== Pull a valid Git workspace update ==\n'
sed -i.bak 's/name: Task 13 Smoke/name: Task 13 Smoke Updated/' \
"$TASK13_SEED/workspaces/task13-smoke.yaml"
rm "$TASK13_SEED/workspaces/task13-smoke.yaml.bak"
task13_run_logged "commit valid workspace update" git -C "$TASK13_SEED" add workspaces/task13-smoke.yaml
task13_run_logged "commit valid workspace update" git -C "$TASK13_SEED" \
-c user.name='Task 13 Smoke' -c user.email='task13-smoke@example.invalid' \
commit -m 'Update Task 13 workspace'
task13_run_logged "push valid workspace update" git -C "$TASK13_SEED" \
push "$TASK13_REMOTE" "HEAD:$TASK13_BRANCH"
chmod -R a+rX "$TASK13_REMOTE"
task13_write_environment /fixtures/remote.git
task13_compose_logged "online Compose recreation" up --detach --force-recreate --wait --wait-timeout 120
task13_compose exec -T core curl --connect-timeout 3 --max-time 15 -fsS -X POST \
http://127.0.0.1:8787/workspace-registry/pull >/dev/null
task13_compose exec -T core curl --connect-timeout 3 --max-time 10 -fsS \
http://127.0.0.1:8787/workspaces \
| grep -Fq 'Task 13 Smoke Updated' || task13_fail "valid Git update was not activated"
valid_head="$(task13_active_registry_head)"
[[ "$valid_head" =~ ^[0-9a-f]{40}$ && "$valid_head" != "$TASK13_INITIAL_HEAD" ]] \
|| task13_fail "valid Git update did not advance the registry head"
TASK13_INITIAL_HEAD="$valid_head"
task13_assert_sentinels
printf '== Reject invalid Git content and retain the valid snapshot ==\n'
printf 'workspace: invalid\n' >"$TASK13_SEED/workspaces/task13-smoke.yaml"
task13_run_logged "commit invalid workspace update" git -C "$TASK13_SEED" add workspaces/task13-smoke.yaml
task13_run_logged "commit invalid workspace update" git -C "$TASK13_SEED" \
-c user.name='Task 13 Smoke' -c user.email='task13-smoke@example.invalid' \
commit -m 'Invalid Task 13 workspace fixture'
task13_run_logged "push invalid workspace update" git -C "$TASK13_SEED" \
push "$TASK13_REMOTE" "HEAD:$TASK13_BRANCH"
chmod -R a+rX "$TASK13_REMOTE"
if task13_compose exec -T core curl --connect-timeout 3 --max-time 15 -fsS -X POST \
http://127.0.0.1:8787/workspace-registry/pull >>"$TASK13_LOG" 2>&1; then
task13_fail "registry accepted invalid Git content"
fi
[[ "$(task13_active_registry_head)" == "$TASK13_INITIAL_HEAD" ]] \
|| task13_fail "invalid Git content replaced the valid registry head"
task13_compose exec -T core curl --connect-timeout 3 --max-time 10 -fsS \
http://127.0.0.1:8787/workspaces \
| grep -Fq 'Task 13 Smoke Updated' || task13_fail "invalid Git content displaced the valid workspace"
task13_assert_sentinels
}
task13_prepare_bad_candidate() {
task13_run_logged "pull pinned dead-core candidate" docker image pull "$TASK13_BAD_CANDIDATE_IMAGE"
TASK13_BAD_CANDIDATE_ID="$(docker image inspect --format '{{.Id}}' "$TASK13_BAD_CANDIDATE_IMAGE")"
[[ "$TASK13_BAD_CANDIDATE_ID" =~ ^sha256:[0-9a-f]{64}$ ]] \
|| task13_fail "bad candidate image identity was not resolved"
}
task13_update_rollback() {
local before_image before_mounts before_head output rc phase after_image after_mounts after_head
printf '== Inject a bad pinned Pi candidate and prove automatic rollback ==\n'
task13_prepare_bad_candidate
before_image="$(docker inspect --format '{{.Image}}' "$(task13_core_id)")"
TASK13_PREVIOUS_IMAGE_ID="$before_image"
before_mounts="$(task13_mount_fingerprint)"
before_head="$(task13_active_registry_head)"
output="$TASK13_TMP/thothctl-update.out"
set +e
"$TASK13_THOTHCTL" --installation "$TASK13_INSTALLATION" pi update \
--version "$TASK13_BAD_PI_VERSION" --source pull --image "$TASK13_BAD_CANDIDATE_IMAGE" --yes \
>"$output" 2>&1
rc=$?
set -e
[[ "$rc" -ne 0 ]] || task13_fail "bad Pi candidate unexpectedly passed update verification"
if grep -Fq "$TASK13_SECRET_VALUE" "$output"; then
task13_fail "thothctl update output exposed the fixture secret"
fi
grep -Fq 'previous core image was restored' "$output" \
|| { task13_sanitize <"$output" >&2; task13_fail "thothctl did not report automatic rollback"; }
[[ -f "$TASK13_UPDATE_STATE" ]] || task13_fail "thothctl update state was not persisted"
phase="$(sed -n 's/.*"phase": "\([^"]*\)".*/\1/p' "$TASK13_UPDATE_STATE" | head -n 1)"
[[ "$phase" == rolled_back ]] || task13_fail "update state phase is not rolled_back"
if grep -Fq "$TASK13_SECRET_VALUE" "$TASK13_UPDATE_STATE"; then
task13_fail "update state exposed the fixture secret"
fi
task13_compose_files
after_image="$(docker inspect --format '{{.Image}}' "$(task13_core_id)")"
after_mounts="$(task13_mount_fingerprint)"
after_head="$(task13_active_registry_head)"
[[ "$after_image" == "$before_image" ]] || task13_fail "rollback did not restore the previous core image"
[[ "$after_mounts" == "$before_mounts" ]] || task13_fail "rollback changed persistence volume identity"
[[ "$after_head" == "$before_head" ]] || task13_fail "rollback changed the active registry revision"
task13_assert_sentinels
task13_run_logged "post-rollback thothctl doctor" \
"$TASK13_THOTHCTL" --installation "$TASK13_INSTALLATION" pi doctor
task13_compose exec -T core curl --connect-timeout 3 --max-time 10 -fsS \
http://127.0.0.1:8787/workspaces \
| grep -Fq 'Task 13 Smoke' || task13_fail "rollback lost the active workspace"
}
task13_remove_labeled_container() {
local name="$1" label
[[ -n "$name" ]] || return 0
if ! docker container inspect "$name" >/dev/null 2>&1; then
return 0
fi
label="$(docker container inspect --format '{{ index .Config.Labels "io.thothii.task13.run" }}' "$name")"
[[ "$label" == "$TASK13_RUN_ID" ]] || {
printf 'refusing to remove foreign container %s\n' "$name" >&2
return 1
}
task13_bounded "$TASK13_CLEANUP_TIMEOUT" "remove owned container" \
docker container rm --force "$name" >/dev/null
}
task13_remove_labeled_image() {
local reference="$1" label
[[ -n "$reference" ]] || return 0
if ! docker image inspect "$reference" >/dev/null 2>&1; then
return 0
fi
label="$(docker image inspect --format '{{ index .Config.Labels "io.thothii.task13.run" }}' "$reference")"
[[ "$label" == "$TASK13_RUN_ID" ]] || {
printf 'refusing to remove foreign image %s\n' "$reference" >&2
return 1
}
task13_bounded "$TASK13_CLEANUP_TIMEOUT" "remove owned image" \
docker image rm "$reference" >/dev/null
}
task13_remove_transaction_image() {
local reference="$1" expected_id="$2" actual_id
[[ -n "$reference" ]] || return 0
if ! docker image inspect "$reference" >/dev/null 2>&1; then
return 0
fi
if [[ ! "$reference" =~ ^thothii-core:thothctl-[0-9a-f]{16}-(candidate|previous)$ \
|| ! "$expected_id" =~ ^sha256:([0-9a-f]{64}|owned)$ ]]; then
printf 'refusing to remove invalid transaction image reference %s\n' "$reference" >&2
return 1
fi
actual_id="$(docker image inspect --format '{{.Id}}' "$reference")"
[[ "$actual_id" == "$expected_id" ]] || {
printf 'refusing to remove foreign transaction image %s\n' "$reference" >&2
return 1
}
task13_bounded "$TASK13_CLEANUP_TIMEOUT" "remove owned transaction image" \
docker image rm "$reference" >/dev/null
}
task13_assert_project_ownership() {
local kind id ids label
for kind in container volume network; do
if ! ids="$(docker "$kind" ls -q --filter "label=com.docker.compose.project=$TASK13_PROJECT")"; then
task13_fail "could not enumerate Compose project $kind resources"
return 1
fi
while IFS= read -r id; do
[[ -n "$id" ]] || continue
case "$kind" in
container)
if ! label="$(docker container inspect --format '{{ index .Config.Labels "io.thothii.task13.run" }}' "$id")"; then
task13_fail "could not inspect Compose project container resource"
return 1
fi
;;
volume)
if ! label="$(docker volume inspect --format '{{ index .Labels "io.thothii.task13.run" }}' "$id")"; then
task13_fail "could not inspect Compose project volume resource"
return 1
fi
;;
network)
if ! label="$(docker network inspect --format '{{ index .Labels "io.thothii.task13.run" }}' "$id")"; then
task13_fail "could not inspect Compose project network resource"
return 1
fi
;;
esac
if [[ "$label" != "$TASK13_RUN_ID" ]]; then
task13_fail "Compose project contains a foreign $kind resource"
return 1
fi
done <<<"$ids"
done
}
task13_assert_built_image_ownership() {
local image label
for image in "$TASK13_CORE_IMAGE" "$TASK13_FRONTEND_IMAGE"; do
label="$(docker image inspect --format '{{ index .Config.Labels "io.thothii.task13.run" }}' "$image")"
[[ "$label" == "$TASK13_RUN_ID" ]] || task13_fail "built image lacks the Task 13 run label"
done
}
task13_cleanup() {
local original_rc="$1" cleanup_rc=0 transaction="" leftovers="" image_id=""
set +e
if [[ "$original_rc" -ne 0 && "${TASK13_FAILURE_LOGGED:-0}" -eq 0 ]] \
&& [[ -n "${TASK13_LOG:-}" && -f "$TASK13_LOG" ]]; then
printf '%s\n' '--- sanitized Task 13 diagnostic log ---' >&2
tail -n 200 "$TASK13_LOG" | task13_sanitize >&2
fi
task13_remove_labeled_container "${TASK13_LLM_CONTAINER:-}" || cleanup_rc=1
if [[ -n "${TASK13_PROJECT:-}" && -n "${TASK13_ROOT:-}" && -f "${TASK13_OVERRIDE:-}" ]]; then
if task13_assert_project_ownership >>"${TASK13_LOG:-/dev/null}" 2>&1; then
task13_compose_files
task13_bounded "$TASK13_CLEANUP_TIMEOUT" "stop owned Compose project" \
"${TASK13_COMPOSE[@]}" down --volumes --remove-orphans --timeout 10 \
>>"${TASK13_LOG:-/dev/null}" 2>&1 || cleanup_rc=1
else
cleanup_rc=1
fi
fi
if [[ -f "${TASK13_UPDATE_STATE:-}" ]]; then
transaction="$(sed -n 's/.*"transaction": "\([^"]*\)".*/\1/p' "$TASK13_UPDATE_STATE" | head -n 1)"
fi
if [[ -n "$transaction" ]]; then
task13_remove_transaction_image "thothii-core:thothctl-$transaction-candidate" \
"${TASK13_BAD_CANDIDATE_ID:-}" >>"${TASK13_LOG:-/dev/null}" 2>&1 || cleanup_rc=1
task13_remove_transaction_image "thothii-core:thothctl-$transaction-previous" \
"${TASK13_PREVIOUS_IMAGE_ID:-}" >>"${TASK13_LOG:-/dev/null}" 2>&1 || cleanup_rc=1
fi
for image in \
"${TASK13_FRONTEND_IMAGE:-}" \
"${TASK13_CORE_IMAGE:-}"; do
[[ -n "$image" ]] || continue
task13_remove_labeled_image "$image" >>"${TASK13_LOG:-/dev/null}" 2>&1 || cleanup_rc=1
done
if [[ -n "${TASK13_RUN_ID:-}" ]]; then
while IFS= read -r image_id; do
[[ -n "$image_id" ]] || continue
task13_remove_labeled_image "$image_id" >>"${TASK13_LOG:-/dev/null}" 2>&1 || cleanup_rc=1
done < <(docker image ls -q --filter "label=io.thothii.task13.run=$TASK13_RUN_ID" | sort -u)
fi
if [[ -n "${TASK13_CONTROL_DIR:-}" ]]; then
if [[ "$TASK13_CONTROL_DIR" == "$TASK13_ROOT/.thothctl/$TASK13_PROJECT" \
&& "$TASK13_PROJECT" =~ ^thothii-[0-9a-f]{12}$ ]]; then
rm -rf "$TASK13_CONTROL_DIR"
else
cleanup_rc=1
fi
fi
if [[ -n "${TASK13_RUN_ID:-}" ]]; then
leftovers="$(docker container ls -aq --filter "label=io.thothii.task13.run=$TASK13_RUN_ID")"
leftovers+="$(docker volume ls -q --filter "label=io.thothii.task13.run=$TASK13_RUN_ID")"
leftovers+="$(docker network ls -q --filter "label=io.thothii.task13.run=$TASK13_RUN_ID")"
leftovers+="$(docker image ls -q --filter "label=io.thothii.task13.run=$TASK13_RUN_ID")"
[[ -z "$leftovers" ]] || cleanup_rc=1
fi
if [[ -n "${TASK13_TMP:-}" && -d "$TASK13_TMP" ]]; then
if [[ "${TASK13_TMP%/*}" == "${TASK13_TMP_PARENT:-}" \
&& "${TASK13_TMP##*/}" == thothii-task13.* ]]; then
rm -rf "$TASK13_TMP"
else
cleanup_rc=1
fi
fi
if [[ "$cleanup_rc" -eq 0 ]]; then
printf 'Task 13 cleanup proof: no labeled containers, volumes, networks, or images remain for %s.\n' \
"${TASK13_RUN_ID:-unknown}"
else
printf 'Task 13 cleanup proof failed for %s.\n' "${TASK13_RUN_ID:-unknown}" >&2
fi
trap - EXIT
if [[ "$original_rc" -ne 0 ]]; then
exit "$original_rc"
fi
exit "$cleanup_rc"
}
task13_self_test_sanitizer() {
local input output leaked
TASK13_SECRET_VALUE="fixture-known-secret"
input="$(printf '%s\n' \
'fixture-known-secret' \
'password=plain-secret' \
'{"api_key":"json-secret"}' \
'Authorization: Bearer bearer-secret' \
'https://alice:url-secret@example.invalid/repo.git')"
output="$(printf '%s\n' "$input" | task13_sanitize)"
for leaked in fixture-known-secret plain-secret json-secret bearer-secret url-secret; do
if grep -Fq "$leaked" <<<"$output"; then
task13_fail "sanitizer leaked $leaked"
fi
done
[[ "$(grep -Fc '[REDACTED]' <<<"$output")" -eq 5 ]] \
|| task13_fail "sanitizer did not redact every credential form"
}
task13_self_test_cleanup_ownership() {
local calls foreign_error owned_name foreign_name
calls="$(mktemp "${TMPDIR:-/tmp}/thothii-task13-cleanup-contract.XXXXXX")"
foreign_error="$calls.foreign-error"
owned_name="task13-owned-contract"
foreign_name="task13-foreign-contract"
TASK13_RUN_ID="task13-contract-run"
docker() {
printf '%s\n' "$*" >>"$calls"
if [[ "$1 $2" == "container inspect" ]]; then
if [[ "$3" == "--format" ]]; then
if [[ "${*: -1}" == "$owned_name" ]]; then
printf '%s\n' "$TASK13_RUN_ID"
else
printf '%s\n' 'some-other-run'
fi
fi
return 0
fi
[[ "$1 $2" == "container rm" ]]
}
if task13_remove_labeled_container "$foreign_name" 2>"$foreign_error"; then
unset -f docker
rm -f "$calls" "$foreign_error"
task13_fail "cleanup accepted a foreign-labeled container"
fi
if grep -Fq "container rm --force $foreign_name" "$calls"; then
unset -f docker
rm -f "$calls" "$foreign_error"
task13_fail "cleanup attempted to remove a foreign-labeled container"
fi
grep -Fq "refusing to remove foreign container $foreign_name" "$foreign_error" \
|| task13_fail "cleanup refusal was not explicit"
task13_remove_labeled_container "$owned_name"
[[ "$(grep -Fc "container rm --force $owned_name" "$calls")" -eq 1 ]] \
|| task13_fail "cleanup did not remove exactly the owned container"
unset -f docker
rm -f "$calls" "$foreign_error"
}
task13_self_test_image_cleanup_ownership() {
local calls foreign_error owned_ref foreign_ref
calls="$(mktemp "${TMPDIR:-/tmp}/thothii-task13-image-cleanup-contract.XXXXXX")"
foreign_error="$calls.foreign-error"
owned_ref="task13-owned-contract:local"
foreign_ref="task13-foreign-contract:local"
TASK13_RUN_ID="task13-contract-run"
if ! declare -F task13_remove_labeled_image >/dev/null; then
rm -f "$calls" "$foreign_error"
task13_fail "image cleanup ownership guard is missing"
fi
docker() {
printf '%s\n' "$*" >>"$calls"
if [[ "$1 $2" == "image inspect" ]]; then
if [[ "$3" == "--format" ]]; then
if [[ "${*: -1}" == "$owned_ref" ]]; then
printf '%s\n' "$TASK13_RUN_ID"
else
printf '%s\n' 'some-other-run'
fi
fi
return 0
fi
[[ "$1 $2" == "image rm" ]]
}
if task13_remove_labeled_image "$foreign_ref" 2>"$foreign_error"; then
unset -f docker
rm -f "$calls" "$foreign_error"
task13_fail "cleanup accepted a foreign-labeled image"
fi
if grep -Fq "image rm $foreign_ref" "$calls"; then
unset -f docker
rm -f "$calls" "$foreign_error"
task13_fail "cleanup attempted to remove a foreign-labeled image"
fi
grep -Fq "refusing to remove foreign image $foreign_ref" "$foreign_error" \
|| task13_fail "image cleanup refusal was not explicit"
task13_remove_labeled_image "$owned_ref"
[[ "$(grep -Fc "image rm $owned_ref" "$calls")" -eq 1 ]] \
|| task13_fail "cleanup did not remove exactly the owned image reference"
unset -f docker
rm -f "$calls" "$foreign_error"
}
task13_self_test_transaction_image_cleanup() {
local calls foreign_error owned_ref foreign_ref
calls="$(mktemp "${TMPDIR:-/tmp}/thothii-task13-transaction-cleanup-contract.XXXXXX")"
foreign_error="$calls.foreign-error"
owned_ref="thothii-core:thothctl-0123456789abcdef-candidate"
foreign_ref="thothii-core:thothctl-fedcba9876543210-candidate"
if ! declare -F task13_remove_transaction_image >/dev/null; then
rm -f "$calls" "$foreign_error"
task13_fail "transaction image cleanup guard is missing"
fi
docker() {
printf '%s\n' "$*" >>"$calls"
if [[ "$1 $2" == "image inspect" ]]; then
if [[ "$3" == "--format" ]]; then
if [[ "${*: -1}" == "$owned_ref" ]]; then
printf '%s\n' 'sha256:owned'
else
printf '%s\n' 'sha256:foreign'
fi
fi
return 0
fi
[[ "$1 $2" == "image rm" ]]
}
if task13_remove_transaction_image "$foreign_ref" 'sha256:owned' 2>"$foreign_error"; then
unset -f docker
rm -f "$calls" "$foreign_error"
task13_fail "cleanup accepted a transaction image with a foreign identity"
fi
if grep -Fq "image rm $foreign_ref" "$calls"; then
unset -f docker
rm -f "$calls" "$foreign_error"
task13_fail "cleanup attempted to remove a foreign transaction image"
fi
grep -Fq "refusing to remove foreign transaction image $foreign_ref" "$foreign_error" \
|| task13_fail "transaction image cleanup refusal was not explicit"
task13_remove_transaction_image "$owned_ref" 'sha256:owned'
[[ "$(grep -Fc "image rm $owned_ref" "$calls")" -eq 1 ]] \
|| task13_fail "cleanup did not remove exactly the owned transaction image reference"
unset -f docker
rm -f "$calls" "$foreign_error"
}
task13_self_test_source_contract() {
local root host_network push_command registry_function workflow uses_count pinned_uses_count
root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)"
workflow="$root/.github/workflows/deployment.yml"
host_network='--network'' host'
push_command='docker image ''push'
registry_function='task13_start_''registry'
if rg -n 'docker[[:space:]]+(system[[:space:]]+)?prune' \
"$root/scripts/unified-deployment-smoke.sh" \
"$root/scripts/thothctl-update-smoke.sh" >/dev/null; then
task13_fail "Task 13 smoke scripts must never prune global Docker state"
fi
! grep -Fq -- "$host_network" "$root/scripts/unified-deployment-smoke.sh" \
|| task13_fail "the image registry must not depend on host networking"
if grep -Fq -- "$push_command" "$root/scripts/unified-deployment-smoke.sh" \
|| grep -Fq -- "$registry_function" "$root/scripts/unified-deployment-smoke.sh"; then
task13_fail "the rollback fixture must not depend on a daemon-to-host local image registry"
fi
grep -Eq '^TASK13_BAD_CANDIDATE_IMAGE="[^"[:space:]]+@sha256:[0-9a-f]{64}"$' \
"$root/scripts/unified-deployment-smoke.sh" \
|| task13_fail "the bad rollback candidate must be an immutable digest reference"
grep -Eq 'timeout .*scripts/unified-deployment-smoke\.sh' "$workflow" \
|| task13_fail "CI lacks an outer timeout for the unified deployment smoke"
grep -Eq 'timeout .*scripts/thothctl-update-smoke\.sh' "$workflow" \
|| task13_fail "CI lacks an outer timeout for the thothctl update smoke"
uses_count="$(grep -Ec '^[[:space:]]+uses:' "$workflow")"
pinned_uses_count="$(grep -Ec '^[[:space:]]+uses: [^[:space:]]+@[0-9a-f]{40}([[:space:]]|$)' "$workflow")"
[[ "$uses_count" -gt 0 && "$uses_count" -eq "$pinned_uses_count" ]] \
|| task13_fail "every deployment workflow action must use a full immutable commit pin"
if grep -Fq '${{ secrets.' "$workflow"; then
task13_fail "the deployment release gate must not require repository secrets"
fi
for command in \
'bash scripts/verify-line-endings.sh' \
'bash scripts/test-unified-compose.sh' \
'bash scripts/test-compose-secret-policy.sh' \
'bash scripts/test-no-deployment-coupling.sh' \
'bash scripts/test-verify-workspace-install-docs.sh' \
'npx vitest run' \
'npx tsc --noEmit -p .' \
'npx tsc -b' \
'./scripts/test-windows-clone-contract.ps1'; do
grep -Fq "$command" "$workflow" || task13_fail "CI coverage is missing: $command"
done
}
task13_self_test() {
task13_self_test_sanitizer
task13_self_test_cleanup_ownership
task13_self_test_image_cleanup_ownership
task13_self_test_transaction_image_cleanup
task13_self_test_source_contract
printf 'Task 13 smoke safety contracts passed.\n'
}
task13_initialize() {
umask 077
TASK13_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)"
TASK13_TMP_PARENT="$(cd "${TMPDIR:-/tmp}" && pwd -P)"
TASK13_TMP="$(mktemp -d "$TASK13_TMP_PARENT/thothii-task13.XXXXXX")"
TASK13_TMP="$(cd "$TASK13_TMP" && pwd -P)"
TASK13_LOG="$TASK13_TMP/task13.log"
TASK13_FAILURE_LOGGED=0
: >"$TASK13_LOG"
trap 'task13_cleanup $?' EXIT
trap 'exit 130' INT TERM HUP
TASK13_RUN_ID="$(date -u +%Y%m%d%H%M%S)-$$-${RANDOM:-0}"
TASK13_INSTALLATION="$TASK13_TMP/thothii-installation.yaml"
TASK13_PROJECT="thothii-$(task13_sha256_text "$TASK13_INSTALLATION" | cut -c1-12)"
TASK13_CONTROL_DIR="$TASK13_ROOT/.thothctl/$TASK13_PROJECT"
[[ ! -e "$TASK13_CONTROL_DIR" ]] || task13_fail "unique thothctl control directory already exists"
TASK13_CURRENT_IMAGE_OVERRIDE="$TASK13_CONTROL_DIR/current-image.yaml"
TASK13_UPDATE_STATE="$TASK13_CONTROL_DIR/update-state.json"
TASK13_REMOTE="$TASK13_TMP/remote.git"
TASK13_SEED="$TASK13_TMP/seed"
TASK13_BRANCH="task13-smoke"
TASK13_ENV_FILE="$TASK13_TMP/local.env"
TASK13_OVERRIDE="$TASK13_TMP/compose.task13.yaml"
TASK13_PI_AUTH="$TASK13_TMP/pi-auth.json"
TASK13_SECRETS="$TASK13_TMP/thothii.secrets"
TASK13_PI_MODELS="$TASK13_TMP/models.json"
TASK13_PI_SETTINGS="$TASK13_TMP/pi-settings.json"
TASK13_LLM_SERVER="$TASK13_TMP/fake-llm.mjs"
TASK13_THOTHCTL_DIR="$TASK13_TMP/thothctl"
TASK13_LLM_CONTAINER="$TASK13_PROJECT-llm"
TASK13_CORE_IMAGE="task13-core-$TASK13_RUN_ID:local"
TASK13_FRONTEND_IMAGE="task13-frontend-$TASK13_RUN_ID:local"
TASK13_SECRET_VALUE="task13-secret-$TASK13_RUN_ID"
TASK13_NETWORK=""
TASK13_BAD_CANDIDATE_ID=""
TASK13_PREVIOUS_IMAGE_ID=""
}
task13_require_tools() {
for command in bash git docker curl sed awk grep rg sort; do
command -v "$command" >/dev/null 2>&1 || task13_fail "$command is required"
done
task13_run_logged "Docker daemon readiness" docker info
task13_run_logged "Docker Compose readiness" docker compose version
task13_self_test_source_contract
}
task13_smoke_main() {
local mode="${1:-full}"
[[ "$mode" == full || "$mode" == update ]] || task13_fail "unknown Task 13 smoke mode: $mode"
task13_initialize
task13_require_tools
task13_write_fixture_files
task13_write_environment /fixtures/remote.git
task13_seed_registry
task13_build_thothctl
task13_start_stack
task13_assert_project_ownership
task13_assert_built_image_ownership
task13_assert_runtime
task13_prepare_persistence
if [[ "$mode" == full ]]; then
task13_registry_lifecycle
fi
task13_update_rollback
printf 'Task 13 %s deployment smoke passed.\n' "$mode"
}
if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
if [[ "${1:-}" == "--self-test" ]]; then
task13_self_test
else
task13_smoke_main full
fi
fi
Reference in New Issue View Git Blame Copy Permalink