fix: complete deployment release gates
This commit is contained in:
@@ -4,11 +4,15 @@
|
||||
# isolated fixture, exact cleanup, and sanitized failure reporting.
|
||||
set -euo pipefail
|
||||
|
||||
TASK13_BAD_CANDIDATE_IMAGE="registry:2.8.3@sha256:a3d8aaa63ed8681a604f1dea0aa03f100d5895b6a58ace528858a7b332415373"
|
||||
TASK13_BAD_CANDIDATE_IMAGE="hello-world@sha256:c3cbe1cc1aa588a64951ac6286e0df7b27fe2e6324b1001c619bb358770c0178"
|
||||
TASK13_BAD_CANDIDATE_BEHAVIOR="stopped"
|
||||
TASK13_BAD_PI_VERSION="0.80.4-task13"
|
||||
TASK13_CURL_CONNECT_TIMEOUT=3
|
||||
TASK13_CURL_MAX_TIME=10
|
||||
TASK13_CLEANUP_TIMEOUT=20
|
||||
TASK13_COMMAND_TIMEOUT=900
|
||||
TASK13_SMOKE_TIMEOUT=1800
|
||||
TASK13_TERM_GRACE=45
|
||||
|
||||
task13_fail() {
|
||||
printf 'Task 13 smoke failed: %s\n' "$*" >&2
|
||||
@@ -49,41 +53,82 @@ task13_log_failure() {
|
||||
task13_run_logged() {
|
||||
local label="$1"
|
||||
shift
|
||||
if ! "$@" >>"$TASK13_LOG" 2>&1; then
|
||||
if ! task13_bounded "$TASK13_COMMAND_TIMEOUT" "$label" "$@" >>"$TASK13_LOG" 2>&1; then
|
||||
task13_log_failure "$label"
|
||||
fi
|
||||
}
|
||||
|
||||
task13_bounded() {
|
||||
local seconds="$1" label="$2" command_pid watchdog_pid rc
|
||||
local seconds="$1" label="$2" command_pid watchdog_pid rc watchdog_rc=0
|
||||
local monitor_enabled=0 timeout_marker command_group
|
||||
shift 2
|
||||
"$@" &
|
||||
command_pid=$!
|
||||
timeout_marker="$(mktemp "${TMPDIR:-/tmp}/thothii-task13-timeout.XXXXXX")"
|
||||
[[ $- == *m* ]] && monitor_enabled=1
|
||||
if [[ -n "${TASK13_ACTIVE_GROUP:-}" ]]; then
|
||||
[[ "$TASK13_ACTIVE_GROUP" =~ ^[0-9]+$ ]] \
|
||||
|| task13_fail "invalid active Task 13 process group"
|
||||
set +m
|
||||
"$@" &
|
||||
command_pid=$!
|
||||
command_group="$TASK13_ACTIVE_GROUP"
|
||||
else
|
||||
set -m
|
||||
"$@" &
|
||||
command_pid=$!
|
||||
command_group="$command_pid"
|
||||
[[ "$monitor_enabled" -eq 1 ]] || set +m
|
||||
fi
|
||||
(
|
||||
local sleep_pid
|
||||
local sleep_pid grace_deadline
|
||||
sleep "$seconds" &
|
||||
sleep_pid=$!
|
||||
trap 'kill "$sleep_pid" 2>/dev/null || true' EXIT INT TERM
|
||||
wait "$sleep_pid" 2>/dev/null || exit 0
|
||||
trap - EXIT INT TERM
|
||||
if kill -0 "$command_pid" 2>/dev/null; then
|
||||
if kill -0 -- "-$command_group" 2>/dev/null; then
|
||||
trap '' TERM
|
||||
printf 'timeout\n' >"$timeout_marker"
|
||||
printf 'Task 13 command timed out after %ss: %s\n' "$seconds" "$label" >&2
|
||||
kill -TERM "$command_pid" 2>/dev/null || true
|
||||
sleep 5
|
||||
kill -KILL "$command_pid" 2>/dev/null || true
|
||||
kill -TERM -- "-$command_group" 2>/dev/null || true
|
||||
grace_deadline=$((SECONDS + TASK13_TERM_GRACE))
|
||||
while kill -0 -- "-$command_group" 2>/dev/null && ((SECONDS < grace_deadline)); do
|
||||
sleep 1
|
||||
done
|
||||
kill -KILL -- "-$command_group" 2>/dev/null || true
|
||||
exit 124
|
||||
fi
|
||||
) &
|
||||
watchdog_pid=$!
|
||||
if [[ -n "${TASK13_ACTIVE_GROUP:-}" && "$monitor_enabled" -eq 1 ]]; then
|
||||
set -m
|
||||
fi
|
||||
if wait "$command_pid"; then
|
||||
rc=0
|
||||
else
|
||||
rc=$?
|
||||
fi
|
||||
kill "$watchdog_pid" 2>/dev/null || true
|
||||
wait "$watchdog_pid" 2>/dev/null || true
|
||||
if [[ -s "$timeout_marker" ]]; then
|
||||
wait "$watchdog_pid" 2>/dev/null || watchdog_rc=$?
|
||||
else
|
||||
kill "$watchdog_pid" 2>/dev/null || true
|
||||
wait "$watchdog_pid" 2>/dev/null || true
|
||||
fi
|
||||
rm -f "$timeout_marker"
|
||||
[[ "$watchdog_rc" -eq 124 ]] && return 124
|
||||
return "$rc"
|
||||
}
|
||||
|
||||
task13_supervised_call() {
|
||||
TASK13_ACTIVE_GROUP="$BASHPID"
|
||||
"$@"
|
||||
}
|
||||
|
||||
task13_supervise() {
|
||||
local seconds="$1" label="$2"
|
||||
shift 2
|
||||
task13_bounded "$seconds" "$label" task13_supervised_call "$@"
|
||||
}
|
||||
|
||||
task13_compose_files() {
|
||||
TASK13_COMPOSE=(
|
||||
docker compose
|
||||
@@ -91,10 +136,20 @@ task13_compose_files() {
|
||||
--project-directory "$TASK13_ROOT"
|
||||
--env-file "$TASK13_ENV_FILE"
|
||||
-f "$TASK13_ROOT/compose.yaml"
|
||||
-f "$TASK13_ROOT/deploy/compose.local.yaml"
|
||||
-f "$TASK13_OVERRIDE"
|
||||
)
|
||||
if [[ -f "$TASK13_CURRENT_IMAGE_OVERRIDE" ]]; then
|
||||
if [[ "${TASK13_PROFILE:-local}" == server ]]; then
|
||||
TASK13_COMPOSE+=(
|
||||
-f "$TASK13_ROOT/deploy/compose.server.yaml"
|
||||
-f "$TASK13_ROOT/deploy/compose.session-server.yaml.example"
|
||||
-f "$TASK13_OVERRIDE"
|
||||
)
|
||||
else
|
||||
TASK13_COMPOSE+=(
|
||||
-f "$TASK13_ROOT/deploy/compose.local.yaml"
|
||||
-f "$TASK13_OVERRIDE"
|
||||
)
|
||||
fi
|
||||
if [[ "${TASK13_PROFILE:-local}" == local && -f "$TASK13_CURRENT_IMAGE_OVERRIDE" ]]; then
|
||||
TASK13_COMPOSE+=(-f "$TASK13_CURRENT_IMAGE_OVERRIDE")
|
||||
fi
|
||||
}
|
||||
@@ -224,6 +279,15 @@ services:
|
||||
PI_THINKING: low
|
||||
THT_WORKSPACE_INSTALLATION_ID: task13-smoke
|
||||
THT_WORKSPACE_REGISTRY_ROOT: /data/workspace-registry
|
||||
THT_WS_TASK13_SMOKE_DWH_HOST: dwh.task13.invalid
|
||||
THT_WS_TASK13_SMOKE_DWH_PORT: "5432"
|
||||
THT_WS_TASK13_SMOKE_DWH_USER: task13_reader
|
||||
THT_WS_TASK13_SMOKE_DWH_PASSWORD_FILE: /run/secrets/thothii.secrets
|
||||
THT_WS_TASK13_SMOKE_VECTOR_HOST: vector.task13.invalid
|
||||
THT_WS_TASK13_SMOKE_VECTOR_PORT: "5432"
|
||||
THT_WS_TASK13_SMOKE_VECTOR_USER: task13_vector_reader
|
||||
THT_WS_TASK13_SMOKE_VECTOR_PASSWORD_FILE: /run/secrets/thothii.secrets
|
||||
THT_WS_TASK13_SMOKE_EMBEDDING_BASE_URL: http://$TASK13_LLM_CONTAINER:9000
|
||||
labels:
|
||||
io.thothii.task13.run: "$TASK13_RUN_ID"
|
||||
volumes: !override
|
||||
@@ -272,6 +336,114 @@ EOF
|
||||
chmod 0600 "$TASK13_INSTALLATION"
|
||||
}
|
||||
|
||||
task13_write_server_fixture_files() {
|
||||
local data_root pi_root registry_root remote_path workspace_path
|
||||
printf '{}\n' >"$TASK13_PI_AUTH"
|
||||
printf 'THT_MODEL_API_KEY=%s\n' "$TASK13_SECRET_VALUE" >"$TASK13_SECRETS"
|
||||
printf '%s\n' "$TASK13_SESSION_PASSWORD" >"$TASK13_SESSION_RUNTIME_PASSWORD"
|
||||
printf '%s\n' "$TASK13_SESSION_MIGRATOR_PASSWORD" >"$TASK13_SESSION_MIGRATOR_PASSWORD_FILE"
|
||||
cat >"$TASK13_SESSION_CA" <<'EOF'
|
||||
-----BEGIN CERTIFICATE-----
|
||||
VEFTSzEzLURJU1BPU0FCTEUtU0VTU0lPTi1DQQ==
|
||||
-----END CERTIFICATE-----
|
||||
EOF
|
||||
chmod 0644 "$TASK13_PI_AUTH" "$TASK13_SESSION_CA"
|
||||
chmod 0600 "$TASK13_SECRETS" "$TASK13_SESSION_RUNTIME_PASSWORD" \
|
||||
"$TASK13_SESSION_MIGRATOR_PASSWORD_FILE"
|
||||
|
||||
cat >"$TASK13_SERVER_WORKSPACE_CONFIG" <<'EOF'
|
||||
language: en
|
||||
session_storage:
|
||||
type: postgres_direct
|
||||
connection:
|
||||
host: ${THT_SESSION_DB_HOST}
|
||||
port: ${THT_SESSION_DB_PORT}
|
||||
database: ${THT_SESSION_DB_NAME}
|
||||
schema: thoth_sessions
|
||||
user: ${THT_SESSION_RUNTIME_USER}
|
||||
password_file: ${THT_SESSION_RUNTIME_PASSWORD_FILE}
|
||||
sslmode: ${THT_SESSION_DB_SSLMODE}
|
||||
sslrootcert: ${THT_SESSION_DB_SSLROOTCERT}
|
||||
roots:
|
||||
artifacts: artifacts
|
||||
indexes: indexes
|
||||
sessions: sessions
|
||||
EOF
|
||||
chmod 0600 "$TASK13_SERVER_WORKSPACE_CONFIG"
|
||||
|
||||
mkdir -p "$TASK13_SERVER_DATA" "$TASK13_SERVER_PI_STATE" "$TASK13_SERVER_REGISTRY"
|
||||
chmod 0777 "$TASK13_SERVER_DATA" "$TASK13_SERVER_PI_STATE" "$TASK13_SERVER_REGISTRY"
|
||||
data_root="$TASK13_SERVER_DATA"
|
||||
pi_root="$TASK13_SERVER_PI_STATE"
|
||||
registry_root="$TASK13_SERVER_REGISTRY"
|
||||
remote_path="$TASK13_REMOTE"
|
||||
workspace_path="$TASK13_SERVER_WORKSPACE_CONFIG"
|
||||
|
||||
cat >"$TASK13_OVERRIDE" <<EOF
|
||||
services:
|
||||
core:
|
||||
image: $TASK13_CORE_IMAGE
|
||||
build:
|
||||
labels:
|
||||
io.thothii.task13.run: "$TASK13_RUN_ID"
|
||||
environment:
|
||||
THT_WS_TASK13_SMOKE_DWH_HOST: dwh.task13.invalid
|
||||
THT_WS_TASK13_SMOKE_DWH_PORT: "5432"
|
||||
THT_WS_TASK13_SMOKE_DWH_USER: task13_reader
|
||||
THT_WS_TASK13_SMOKE_DWH_PASSWORD_FILE: /run/secrets/thothii.secrets
|
||||
THT_WS_TASK13_SMOKE_VECTOR_HOST: vector.task13.invalid
|
||||
THT_WS_TASK13_SMOKE_VECTOR_PORT: "5432"
|
||||
THT_WS_TASK13_SMOKE_VECTOR_USER: task13_vector_reader
|
||||
THT_WS_TASK13_SMOKE_VECTOR_PASSWORD_FILE: /run/secrets/thothii.secrets
|
||||
THT_WS_TASK13_SMOKE_EMBEDDING_BASE_URL: https://embedding.task13.invalid
|
||||
labels:
|
||||
io.thothii.task13.run: "$TASK13_RUN_ID"
|
||||
volumes:
|
||||
- $remote_path:/fixtures/remote.git:ro
|
||||
frontend:
|
||||
image: $TASK13_FRONTEND_IMAGE
|
||||
build:
|
||||
labels:
|
||||
io.thothii.task13.run: "$TASK13_RUN_ID"
|
||||
labels:
|
||||
io.thothii.task13.run: "$TASK13_RUN_ID"
|
||||
session-migrate:
|
||||
image: $TASK13_CORE_IMAGE
|
||||
networks:
|
||||
thothii:
|
||||
labels:
|
||||
io.thothii.task13.run: "$TASK13_RUN_ID"
|
||||
EOF
|
||||
chmod 0600 "$TASK13_OVERRIDE"
|
||||
|
||||
{
|
||||
printf 'THOTH_HTTP_PORT=0\n'
|
||||
printf 'THOTH_SERVER_BIND=127.0.0.1\n'
|
||||
printf 'MAX_PI_PROCESSES=2\n'
|
||||
printf 'PI_AUTH_FILE=%s\n' "$TASK13_PI_AUTH"
|
||||
printf 'THT_SECRETS_FILE=%s\n' "$TASK13_SECRETS"
|
||||
printf 'THT_WORKSPACE_GIT_REMOTE=/fixtures/remote.git\n'
|
||||
printf 'THT_WORKSPACE_GIT_BRANCH=%s\n' "$TASK13_BRANCH"
|
||||
printf 'THT_WORKSPACE_GIT_AUTHOR_NAME=Task 13 Server Smoke\n'
|
||||
printf 'THT_WORKSPACE_GIT_AUTHOR_EMAIL=task13-server@example.invalid\n'
|
||||
printf 'THT_DATA_ROOT=%s\n' "$data_root"
|
||||
printf 'THT_PI_STATE_ROOT=%s\n' "$pi_root"
|
||||
printf 'THT_WORKSPACE_REGISTRY_ROOT=%s\n' "$registry_root"
|
||||
printf 'THT_SERVER_WORKSPACE_CONFIG=%s\n' "$workspace_path"
|
||||
printf 'THT_SESSION_DB_HOST=task13-session.invalid\n'
|
||||
printf 'THT_SESSION_DB_PORT=5432\n'
|
||||
printf 'THT_SESSION_DB_NAME=task13\n'
|
||||
printf 'THT_SESSION_RUNTIME_USER=task13_runtime\n'
|
||||
printf 'THT_SESSION_MIGRATOR_USER=task13_migrator\n'
|
||||
printf 'THT_SESSION_DB_SSLMODE=verify-full\n'
|
||||
printf 'THT_SESSION_RUNTIME_PASSWORD_SOURCE=%s\n' "$TASK13_SESSION_RUNTIME_PASSWORD"
|
||||
printf 'THT_SESSION_MIGRATOR_PASSWORD_SOURCE=%s\n' "$TASK13_SESSION_MIGRATOR_PASSWORD_FILE"
|
||||
printf 'THT_SESSION_CA_SOURCE=%s\n' "$TASK13_SESSION_CA"
|
||||
printf 'THT_LLM_URL=https://llm.task13.invalid/v1\n'
|
||||
} >"$TASK13_ENV_FILE"
|
||||
chmod 0600 "$TASK13_ENV_FILE"
|
||||
}
|
||||
|
||||
task13_seed_registry() {
|
||||
mkdir -p "$TASK13_SEED/workspaces"
|
||||
task13_run_logged "initialize bare workspace registry" \
|
||||
@@ -377,6 +549,14 @@ task13_start_stack() {
|
||||
task13_log_failure "deterministic local LLM fixture readiness"
|
||||
}
|
||||
|
||||
task13_start_server_stack() {
|
||||
printf '== Build and start isolated Linux server profile ==\n'
|
||||
task13_assert_rendered_contract
|
||||
task13_compose_logged "build server Compose images" build --pull core frontend
|
||||
task13_compose_logged "start server Compose distribution" \
|
||||
up --detach --wait --wait-timeout 120 core frontend
|
||||
}
|
||||
|
||||
task13_frontend_address() {
|
||||
task13_compose port frontend 8080 | awk 'NR == 1 {print $0}'
|
||||
}
|
||||
@@ -422,6 +602,67 @@ task13_assert_runtime() {
|
||||
task13_run_logged "thothctl Pi doctor" "$TASK13_THOTHCTL" --installation "$TASK13_INSTALLATION" pi doctor
|
||||
}
|
||||
|
||||
task13_assert_server_runtime() {
|
||||
local frontend unauthenticated authenticated session_status core_id frontend_id
|
||||
local expected_core_image expected_frontend_image
|
||||
frontend="$(task13_frontend_address)"
|
||||
task13_run_logged "server frontend health" curl \
|
||||
--connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" --max-time "$TASK13_CURL_MAX_TIME" \
|
||||
--fail --silent --show-error "http://$frontend/"
|
||||
task13_run_logged "server same-origin core health" curl \
|
||||
--connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" --max-time "$TASK13_CURL_MAX_TIME" \
|
||||
--fail --silent --show-error "http://$frontend/api/health"
|
||||
|
||||
core_id="$(task13_core_id)"
|
||||
frontend_id="$(task13_compose ps -q frontend)"
|
||||
expected_core_image="$(docker image inspect --format '{{.Id}}' "$TASK13_CORE_IMAGE")"
|
||||
expected_frontend_image="$(docker image inspect --format '{{.Id}}' "$TASK13_FRONTEND_IMAGE")"
|
||||
[[ "$(docker inspect --format '{{.Image}}' "$core_id")" == "$expected_core_image" ]] \
|
||||
|| task13_fail "server core did not use the smoke-built core image"
|
||||
[[ "$(docker inspect --format '{{.Image}}' "$frontend_id")" == "$expected_frontend_image" ]] \
|
||||
|| task13_fail "server frontend did not use the smoke-built frontend image"
|
||||
task13_compose exec -T core sh -ceu '
|
||||
test "$AUTH_MODE" = upstream
|
||||
test "$THT_SESSION_STORAGE" = postgres
|
||||
test -r /run/secrets/thothii.secrets
|
||||
test -r /run/secrets/session_runtime_password
|
||||
test -r /run/secrets/session_ca.pem
|
||||
test -r /app/harness/workspaces/server-sessions.yaml
|
||||
'
|
||||
task13_mount_fingerprint | grep -Fq '/data = bind :' \
|
||||
|| task13_fail "server profile did not bind the disposable data root"
|
||||
task13_mount_fingerprint | grep -Fq '/home/thoth/.pi = bind :' \
|
||||
|| task13_fail "server profile did not bind the disposable Pi state root"
|
||||
task13_mount_fingerprint | grep -Fq '/data/workspace-registry = bind :' \
|
||||
|| task13_fail "server profile did not bind the disposable registry root"
|
||||
|
||||
unauthenticated="$(curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" \
|
||||
--max-time "$TASK13_CURL_MAX_TIME" --silent --output /dev/null --write-out '%{http_code}' \
|
||||
"http://$frontend/api/workspaces")"
|
||||
[[ "$unauthenticated" == 401 ]] || task13_fail "server profile did not enforce upstream auth"
|
||||
authenticated="$TASK13_TMP/server-workspaces.out"
|
||||
curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" --max-time "$TASK13_CURL_MAX_TIME" \
|
||||
--fail --silent --show-error \
|
||||
-H 'x-thoth-principal-issuer: task13-proxy' \
|
||||
-H 'x-thoth-principal-subject: task13-user' \
|
||||
-H 'x-thoth-principal-display-name: Task 13 User' \
|
||||
"http://$frontend/api/workspaces" >"$authenticated"
|
||||
grep -Fq 'Task 13 Smoke' "$authenticated" \
|
||||
|| task13_fail "authenticated server route did not expose the disposable registry"
|
||||
|
||||
session_status="$(curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" \
|
||||
--max-time "$TASK13_CURL_MAX_TIME" --silent --output "$TASK13_TMP/server-sessions.out" \
|
||||
--write-out '%{http_code}' \
|
||||
-H 'x-thoth-principal-issuer: task13-proxy' \
|
||||
-H 'x-thoth-principal-subject: task13-user' \
|
||||
"http://$frontend/api/sessions")"
|
||||
[[ "$session_status" == 503 ]] \
|
||||
|| task13_fail "disposable unavailable session dependency did not fail closed with 503"
|
||||
if grep -Fq "$TASK13_SECRET_VALUE" "$TASK13_TMP/server-sessions.out"; then
|
||||
task13_fail "server session failure exposed the fixture secret"
|
||||
fi
|
||||
}
|
||||
|
||||
task13_registry_status() {
|
||||
task13_compose exec -T core curl --connect-timeout 3 --max-time 10 -fsS \
|
||||
http://127.0.0.1:8787/workspace-registry/status
|
||||
@@ -523,10 +764,18 @@ task13_registry_lifecycle() {
|
||||
}
|
||||
|
||||
task13_prepare_bad_candidate() {
|
||||
task13_run_logged "pull pinned dead-core candidate" docker image pull "$TASK13_BAD_CANDIDATE_IMAGE"
|
||||
task13_run_logged "pull pinned stopped-core candidate" docker image pull "$TASK13_BAD_CANDIDATE_IMAGE"
|
||||
TASK13_BAD_CANDIDATE_ID="$(docker image inspect --format '{{.Id}}' "$TASK13_BAD_CANDIDATE_IMAGE")"
|
||||
[[ "$TASK13_BAD_CANDIDATE_ID" =~ ^sha256:[0-9a-f]{64}$ ]] \
|
||||
|| task13_fail "bad candidate image identity was not resolved"
|
||||
task13_run_logged "prove bad candidate exits" docker run \
|
||||
--name "$TASK13_BAD_CANDIDATE_CONTAINER" \
|
||||
--label "io.thothii.task13.run=$TASK13_RUN_ID" \
|
||||
"$TASK13_BAD_CANDIDATE_IMAGE"
|
||||
[[ "$(docker container inspect --format '{{.State.Running}}:{{.State.ExitCode}}' \
|
||||
"$TASK13_BAD_CANDIDATE_CONTAINER")" == false:0 ]] \
|
||||
|| task13_fail "bad candidate did not reach the guaranteed stopped state"
|
||||
task13_remove_labeled_container "$TASK13_BAD_CANDIDATE_CONTAINER"
|
||||
}
|
||||
|
||||
task13_update_rollback() {
|
||||
@@ -624,7 +873,12 @@ task13_remove_transaction_image() {
|
||||
task13_assert_project_ownership() {
|
||||
local kind id ids label
|
||||
for kind in container volume network; do
|
||||
if ! ids="$(docker "$kind" ls -q --filter "label=com.docker.compose.project=$TASK13_PROJECT")"; then
|
||||
if [[ "$kind" == container ]]; then
|
||||
if ! ids="$(docker container ls -aq --filter "label=com.docker.compose.project=$TASK13_PROJECT")"; then
|
||||
task13_fail "could not enumerate Compose project container resources"
|
||||
return 1
|
||||
fi
|
||||
elif ! ids="$(docker "$kind" ls -q --filter "label=com.docker.compose.project=$TASK13_PROJECT")"; then
|
||||
task13_fail "could not enumerate Compose project $kind resources"
|
||||
return 1
|
||||
fi
|
||||
@@ -674,6 +928,7 @@ task13_cleanup() {
|
||||
printf '%s\n' '--- sanitized Task 13 diagnostic log ---' >&2
|
||||
tail -n 200 "$TASK13_LOG" | task13_sanitize >&2
|
||||
fi
|
||||
task13_remove_labeled_container "${TASK13_BAD_CANDIDATE_CONTAINER:-}" || cleanup_rc=1
|
||||
task13_remove_labeled_container "${TASK13_LLM_CONTAINER:-}" || cleanup_rc=1
|
||||
if [[ -n "${TASK13_PROJECT:-}" && -n "${TASK13_ROOT:-}" && -f "${TASK13_OVERRIDE:-}" ]]; then
|
||||
if task13_assert_project_ownership >>"${TASK13_LOG:-/dev/null}" 2>&1; then
|
||||
@@ -899,6 +1154,177 @@ task13_self_test_transaction_image_cleanup() {
|
||||
rm -f "$calls" "$foreign_error"
|
||||
}
|
||||
|
||||
task13_self_test_rollback_fixture_contract() {
|
||||
[[ "${TASK13_BAD_CANDIDATE_BEHAVIOR:-}" == stopped ]] \
|
||||
|| task13_fail "rollback candidate is not declared as guaranteed stopped"
|
||||
[[ "$TASK13_BAD_CANDIDATE_IMAGE" =~ ^hello-world@sha256:[0-9a-f]{64}$ ]] \
|
||||
|| task13_fail "rollback candidate is not the digest-pinned stopped fixture"
|
||||
}
|
||||
|
||||
task13_self_test_runtime_binding_fixture() {
|
||||
local fixture_source
|
||||
fixture_source="$(declare -f task13_write_fixture_files)"
|
||||
for variable in \
|
||||
THT_WS_TASK13_SMOKE_DWH_HOST \
|
||||
THT_WS_TASK13_SMOKE_DWH_PORT \
|
||||
THT_WS_TASK13_SMOKE_DWH_USER \
|
||||
THT_WS_TASK13_SMOKE_DWH_PASSWORD_FILE \
|
||||
THT_WS_TASK13_SMOKE_VECTOR_HOST \
|
||||
THT_WS_TASK13_SMOKE_VECTOR_PORT \
|
||||
THT_WS_TASK13_SMOKE_VECTOR_USER \
|
||||
THT_WS_TASK13_SMOKE_VECTOR_PASSWORD_FILE \
|
||||
THT_WS_TASK13_SMOKE_EMBEDDING_BASE_URL; do
|
||||
grep -Fq "$variable" <<<"$fixture_source" \
|
||||
|| task13_fail "rollback fixture lacks runtime binding: $variable"
|
||||
done
|
||||
}
|
||||
|
||||
task13_self_test_server_runtime_binding_fixture() {
|
||||
local fixture_source
|
||||
fixture_source="$(declare -f task13_write_server_fixture_files)"
|
||||
for variable in \
|
||||
THT_WS_TASK13_SMOKE_DWH_HOST \
|
||||
THT_WS_TASK13_SMOKE_DWH_PASSWORD_FILE \
|
||||
THT_WS_TASK13_SMOKE_VECTOR_HOST \
|
||||
THT_WS_TASK13_SMOKE_VECTOR_PASSWORD_FILE \
|
||||
THT_WS_TASK13_SMOKE_EMBEDDING_BASE_URL; do
|
||||
grep -Fq "$variable" <<<"$fixture_source" \
|
||||
|| task13_fail "server fixture lacks runtime binding: $variable"
|
||||
done
|
||||
}
|
||||
|
||||
task13_self_test_stopped_project_containers() {
|
||||
local calls foreign_error
|
||||
calls="$(mktemp "${TMPDIR:-/tmp}/thothii-task13-project-containers.XXXXXX")"
|
||||
foreign_error="$calls.foreign-error"
|
||||
TASK13_PROJECT="thothii-0123456789ab"
|
||||
TASK13_RUN_ID="task13-contract-run"
|
||||
|
||||
docker() {
|
||||
printf '%s\n' "$*" >>"$calls"
|
||||
case "$1 $2 $3" in
|
||||
"container ls -aq") printf '%s\n' stopped-foreign ;;
|
||||
"container inspect --format") printf '%s\n' some-other-run ;;
|
||||
"volume ls -q"|"network ls -q") : ;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
if task13_assert_project_ownership 2>"$foreign_error"; then
|
||||
unset -f docker
|
||||
rm -f "$calls" "$foreign_error"
|
||||
task13_fail "project cleanup accepted a stopped foreign container"
|
||||
fi
|
||||
grep -Fq 'container ls -aq' "$calls" \
|
||||
|| task13_fail "project cleanup did not enumerate stopped containers"
|
||||
grep -Fq 'Compose project contains a foreign container resource' "$foreign_error" \
|
||||
|| task13_fail "stopped foreign container refusal was not explicit"
|
||||
|
||||
: >"$calls"
|
||||
docker() {
|
||||
printf '%s\n' "$*" >>"$calls"
|
||||
case "$1 $2 $3" in
|
||||
"container ls -aq") printf '%s\n' stopped-owned ;;
|
||||
"container inspect --format") printf '%s\n' "$TASK13_RUN_ID" ;;
|
||||
"volume ls -q"|"network ls -q") : ;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
}
|
||||
task13_assert_project_ownership
|
||||
[[ "$(grep -Fc 'container inspect --format' "$calls")" -eq 1 ]] \
|
||||
|| task13_fail "project cleanup did not inspect exactly the stopped owned container"
|
||||
unset -f docker
|
||||
rm -f "$calls" "$foreign_error"
|
||||
}
|
||||
|
||||
task13_self_test_timeout_process_group() {
|
||||
local child_file child_pid="" rc
|
||||
child_file="$(mktemp "${TMPDIR:-/tmp}/thothii-task13-timeout-child.XXXXXX")"
|
||||
set +e
|
||||
task13_bounded 1 "child-process regression" bash -c \
|
||||
'sleep 30 & printf "%s\n" "$!" >"$1"; wait' _ "$child_file" >/dev/null 2>&1
|
||||
rc=$?
|
||||
set -e
|
||||
child_pid="$(sed -n '1p' "$child_file")"
|
||||
[[ "$rc" -ne 0 ]] || {
|
||||
rm -f "$child_file"
|
||||
task13_fail "timed command unexpectedly succeeded"
|
||||
}
|
||||
if [[ -n "$child_pid" ]] && kill -0 "$child_pid" 2>/dev/null; then
|
||||
kill -KILL "$child_pid" 2>/dev/null || true
|
||||
rm -f "$child_file"
|
||||
task13_fail "timed command left its child process alive"
|
||||
fi
|
||||
rm -f "$child_file"
|
||||
}
|
||||
|
||||
task13_self_test_nested_timeout_process_group() {
|
||||
local child_file child_pid="" rc
|
||||
child_file="$(mktemp "${TMPDIR:-/tmp}/thothii-task13-nested-timeout.XXXXXX")"
|
||||
task13_nested_timeout_fixture() {
|
||||
task13_bounded 30 "nested child-process regression" bash -c \
|
||||
'sleep 30 & printf "%s\n" "$!" >"$1"; wait' _ "$child_file"
|
||||
}
|
||||
set +e
|
||||
task13_supervise 1 "nested timeout supervisor" task13_nested_timeout_fixture >/dev/null 2>&1
|
||||
rc=$?
|
||||
set -e
|
||||
unset -f task13_nested_timeout_fixture
|
||||
child_pid="$(sed -n '1p' "$child_file")"
|
||||
[[ "$rc" -ne 0 ]] || {
|
||||
rm -f "$child_file"
|
||||
task13_fail "nested timed command unexpectedly succeeded"
|
||||
}
|
||||
if [[ -n "$child_pid" ]] && kill -0 "$child_pid" 2>/dev/null; then
|
||||
kill -KILL "$child_pid" 2>/dev/null || true
|
||||
rm -f "$child_file"
|
||||
task13_fail "outer timeout left its nested command child alive"
|
||||
fi
|
||||
rm -f "$child_file"
|
||||
}
|
||||
|
||||
task13_self_test_public_timeout_contract() {
|
||||
local root
|
||||
root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)"
|
||||
grep -Eq 'task13_supervise[[:space:]].*task13_smoke_main[[:space:]]+full' \
|
||||
"$root/scripts/unified-deployment-smoke.sh" \
|
||||
|| task13_fail "direct unified smoke invocation lacks an internal supervisor"
|
||||
grep -Eq 'task13_supervise[[:space:]].*task13_smoke_main[[:space:]]+update' \
|
||||
"$root/scripts/thothctl-update-smoke.sh" \
|
||||
|| task13_fail "direct update smoke invocation lacks an internal supervisor"
|
||||
}
|
||||
|
||||
task13_self_test_windows_release_contract() {
|
||||
local root script workflow
|
||||
root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)"
|
||||
script="$root/scripts/test-windows-clone-contract.ps1"
|
||||
workflow="$root/.github/workflows/deployment.yml"
|
||||
grep -Fq 'Task 13 path with spaces' "$script" \
|
||||
|| task13_fail "Windows contract does not operate from a path containing spaces"
|
||||
grep -Fq 'DockerStartup' "$script" \
|
||||
|| task13_fail "Windows contract lacks an explicit Docker startup mode"
|
||||
grep -Fq 'Kill($true)' "$script" \
|
||||
|| task13_fail "Windows bounded runner does not kill the full process tree"
|
||||
grep -Fq 'docker-desktop' "$workflow" \
|
||||
|| task13_fail "workflow lacks a manual self-hosted Windows Docker Desktop gate"
|
||||
grep -Fq -- '-DockerStartup' "$workflow" \
|
||||
|| task13_fail "manual Windows release job does not execute Docker startup mode"
|
||||
}
|
||||
|
||||
task13_self_test_server_release_contract() {
|
||||
local root workflow server_smoke
|
||||
root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)"
|
||||
workflow="$root/.github/workflows/deployment.yml"
|
||||
server_smoke="$root/scripts/server-deployment-smoke.sh"
|
||||
[[ -x "$server_smoke" ]] || task13_fail "bounded Linux server deployment smoke is missing"
|
||||
grep -Fq 'deploy/compose.server.yaml' "$root/scripts/unified-deployment-smoke.sh" \
|
||||
|| task13_fail "server smoke does not load the server profile"
|
||||
grep -Fq 'deploy/compose.session-server.yaml.example' "$root/scripts/unified-deployment-smoke.sh" \
|
||||
|| task13_fail "server smoke does not load the required session overlay"
|
||||
grep -Eq 'timeout .*scripts/server-deployment-smoke\.sh' "$workflow" \
|
||||
|| task13_fail "workflow lacks an outer timeout for the Linux server smoke"
|
||||
}
|
||||
|
||||
task13_self_test_source_contract() {
|
||||
local root host_network push_command registry_function workflow uses_count pinned_uses_count
|
||||
root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)"
|
||||
@@ -950,10 +1376,34 @@ task13_self_test() {
|
||||
task13_self_test_cleanup_ownership
|
||||
task13_self_test_image_cleanup_ownership
|
||||
task13_self_test_transaction_image_cleanup
|
||||
task13_self_test_rollback_fixture_contract
|
||||
task13_self_test_runtime_binding_fixture
|
||||
task13_self_test_server_runtime_binding_fixture
|
||||
task13_self_test_stopped_project_containers
|
||||
task13_self_test_timeout_process_group
|
||||
task13_self_test_nested_timeout_process_group
|
||||
task13_self_test_public_timeout_contract
|
||||
task13_self_test_windows_release_contract
|
||||
task13_self_test_server_release_contract
|
||||
task13_self_test_source_contract
|
||||
printf 'Task 13 smoke safety contracts passed.\n'
|
||||
}
|
||||
|
||||
task13_self_test_case() {
|
||||
case "$1" in
|
||||
rollback) task13_self_test_rollback_fixture_contract ;;
|
||||
runtime-bindings) task13_self_test_runtime_binding_fixture ;;
|
||||
server-bindings) task13_self_test_server_runtime_binding_fixture ;;
|
||||
cleanup) task13_self_test_stopped_project_containers ;;
|
||||
timeout-group) task13_self_test_timeout_process_group ;;
|
||||
timeout-nested) task13_self_test_nested_timeout_process_group ;;
|
||||
timeout-public) task13_self_test_public_timeout_contract ;;
|
||||
windows) task13_self_test_windows_release_contract ;;
|
||||
server) task13_self_test_server_release_contract ;;
|
||||
*) task13_fail "unknown Task 13 self-test case: $1" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
task13_initialize() {
|
||||
umask 077
|
||||
TASK13_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)"
|
||||
@@ -966,6 +1416,7 @@ task13_initialize() {
|
||||
trap 'task13_cleanup $?' EXIT
|
||||
trap 'exit 130' INT TERM HUP
|
||||
TASK13_RUN_ID="$(date -u +%Y%m%d%H%M%S)-$$-${RANDOM:-0}"
|
||||
TASK13_PROFILE="local"
|
||||
TASK13_INSTALLATION="$TASK13_TMP/thothii-installation.yaml"
|
||||
TASK13_PROJECT="thothii-$(task13_sha256_text "$TASK13_INSTALLATION" | cut -c1-12)"
|
||||
TASK13_CONTROL_DIR="$TASK13_ROOT/.thothctl/$TASK13_PROJECT"
|
||||
@@ -984,12 +1435,22 @@ task13_initialize() {
|
||||
TASK13_LLM_SERVER="$TASK13_TMP/fake-llm.mjs"
|
||||
TASK13_THOTHCTL_DIR="$TASK13_TMP/thothctl"
|
||||
TASK13_LLM_CONTAINER="$TASK13_PROJECT-llm"
|
||||
TASK13_BAD_CANDIDATE_CONTAINER="$TASK13_PROJECT-bad-candidate"
|
||||
TASK13_CORE_IMAGE="task13-core-$TASK13_RUN_ID:local"
|
||||
TASK13_FRONTEND_IMAGE="task13-frontend-$TASK13_RUN_ID:local"
|
||||
TASK13_SECRET_VALUE="task13-secret-$TASK13_RUN_ID"
|
||||
TASK13_NETWORK=""
|
||||
TASK13_BAD_CANDIDATE_ID=""
|
||||
TASK13_PREVIOUS_IMAGE_ID=""
|
||||
TASK13_SERVER_DATA="$TASK13_TMP/Server Data"
|
||||
TASK13_SERVER_PI_STATE="$TASK13_TMP/Server Pi State"
|
||||
TASK13_SERVER_REGISTRY="$TASK13_TMP/Server Registry"
|
||||
TASK13_SERVER_WORKSPACE_CONFIG="$TASK13_TMP/server-sessions.yaml"
|
||||
TASK13_SESSION_RUNTIME_PASSWORD="$TASK13_TMP/session-runtime-password"
|
||||
TASK13_SESSION_MIGRATOR_PASSWORD_FILE="$TASK13_TMP/session-migrator-password"
|
||||
TASK13_SESSION_CA="$TASK13_TMP/session-ca.pem"
|
||||
TASK13_SESSION_PASSWORD="task13-runtime-$TASK13_RUN_ID"
|
||||
TASK13_SESSION_MIGRATOR_PASSWORD="task13-migrator-$TASK13_RUN_ID"
|
||||
}
|
||||
|
||||
task13_require_tools() {
|
||||
@@ -1022,10 +1483,26 @@ task13_smoke_main() {
|
||||
printf 'Task 13 %s deployment smoke passed.\n' "$mode"
|
||||
}
|
||||
|
||||
task13_server_smoke_main() {
|
||||
task13_initialize
|
||||
TASK13_PROFILE="server"
|
||||
task13_require_tools
|
||||
task13_write_server_fixture_files
|
||||
task13_seed_registry
|
||||
task13_start_server_stack
|
||||
task13_assert_project_ownership
|
||||
task13_assert_built_image_ownership
|
||||
task13_assert_server_runtime
|
||||
printf 'Task 13 Linux server deployment smoke passed.\n'
|
||||
}
|
||||
|
||||
if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
|
||||
if [[ "${1:-}" == "--self-test" ]]; then
|
||||
task13_self_test
|
||||
elif [[ "${1:-}" == "--self-test-case" ]]; then
|
||||
[[ -n "${2:-}" ]] || task13_fail "--self-test-case requires a case name"
|
||||
task13_self_test_case "$2"
|
||||
else
|
||||
task13_smoke_main full
|
||||
task13_supervise "$TASK13_SMOKE_TIMEOUT" "unified deployment smoke" task13_smoke_main full
|
||||
fi
|
||||
fi
|
||||
|
||||
Reference in New Issue
Block a user