mptyl/ThothII
1
0
Fork 0
Code Issues 25 Pull Requests Actions Packages Projects Releases 1 Wiki Activity
Files
0d0c15b4b8b97d24aaccad7cf8f2719f475b2aac
ThothII/scripts/unified-deployment-smoke.sh
T

2155 lines
93 KiB
Bash
Executable File
Raw Blame History

#!/usr/bin/env bash
# End-to-end release gate for the canonical two-service Compose distribution.
# This file is also sourced by tht-update-smoke.sh so both entry points use the same
# isolated fixture, exact cleanup, and sanitized failure reporting.
set -euo pipefail
TASK13_BAD_CANDIDATE_IMAGE="hello-world@sha256:c3cbe1cc1aa588a64951ac6286e0df7b27fe2e6324b1001c619bb358770c0178"
TASK13_BAD_CANDIDATE_BEHAVIOR="stopped"
TASK13_BAD_PI_VERSION="0.80.4-task13"
TASK13_CURL_CONNECT_TIMEOUT=3
TASK13_CURL_MAX_TIME=10
TASK13_CLEANUP_TIMEOUT=20
TASK13_COMMAND_TIMEOUT=900
TASK13_SMOKE_TIMEOUT=1800
TASK13_TERM_GRACE=45
task13_fail() {
printf 'Task 13 smoke failed: %s\n' "$*" >&2
return 1
}
task13_sha256_text() {
if command -v sha256sum >/dev/null 2>&1; then
printf '%s' "$1" | sha256sum | awk '{print $1}'
elif command -v shasum >/dev/null 2>&1; then
printf '%s' "$1" | shasum -a 256 | awk '{print $1}'
else
task13_fail "sha256sum or shasum is required"
fi
}
task13_sanitize() {
local line
while IFS= read -r line || [[ -n "$line" ]]; do
if [[ -n "${TASK13_SECRET_VALUE:-}" ]]; then
line="${line//"$TASK13_SECRET_VALUE"/[REDACTED]}"
fi
printf '%s\n' "$line"
done | sed -E \
-e 's#([[:alpha:]][[:alnum:]+.-]*://[^:/@[:space:]]+:)[^@/[:space:]]+@#\1[REDACTED]@#g' \
-e 's/(([Pp]roxy-)?[Aa]uthorization:[[:space:]]*([Bb]earer|[Bb]asic)[[:space:]]+)[^[:space:]]+/\1[REDACTED]/g' \
-e "s/(([\"']?[[:alnum:]_.-]*(password|token|api[_-]?key|secret|key)[[:alnum:]_.-]*[\"']?[[:space:]]*[:=][[:space:]]*)([\"'][^\"']*[\"']|[^[:space:],;]+))/\2[REDACTED]/Ig"
}
task13_log_failure() {
local label="$1"
TASK13_FAILURE_LOGGED=1
printf 'Task 13 command failed: %s\n' "$label" >&2
tail -n 200 "$TASK13_LOG" | task13_sanitize >&2
return 1
}
task13_run_logged() {
local label="$1"
shift
if ! task13_bounded "$TASK13_COMMAND_TIMEOUT" "$label" "$@" >>"$TASK13_LOG" 2>&1; then
task13_log_failure "$label"
fi
}
task13_bounded() {
local seconds="$1" label="$2" command_pid watchdog_pid rc watchdog_rc=0
local monitor_enabled=0 timeout_marker command_group
shift 2
timeout_marker="$(mktemp "${TMPDIR:-/tmp}/thothii-task13-timeout.XXXXXX")"
[[ $- == *m* ]] && monitor_enabled=1
if [[ -n "${TASK13_ACTIVE_GROUP:-}" ]]; then
[[ "$TASK13_ACTIVE_GROUP" =~ ^[0-9]+$ ]] \
|| task13_fail "invalid active Task 13 process group"
set +m
"$@" &
command_pid=$!
command_group="$TASK13_ACTIVE_GROUP"
else
set -m
"$@" &
command_pid=$!
command_group="$command_pid"
[[ "$monitor_enabled" -eq 1 ]] || set +m
fi
(
local sleep_pid grace_deadline
sleep "$seconds" &
sleep_pid=$!
trap 'kill "$sleep_pid" 2>/dev/null || true' EXIT INT TERM
wait "$sleep_pid" 2>/dev/null || exit 0
trap - EXIT INT TERM
if kill -0 -- "-$command_group" 2>/dev/null; then
trap '' TERM
printf 'timeout\n' >"$timeout_marker"
printf 'Task 13 command timed out after %ss: %s\n' "$seconds" "$label" >&2
kill -TERM -- "-$command_group" 2>/dev/null || true
grace_deadline=$((SECONDS + TASK13_TERM_GRACE))
while kill -0 -- "-$command_group" 2>/dev/null && ((SECONDS < grace_deadline)); do
sleep 1
done
kill -KILL -- "-$command_group" 2>/dev/null || true
exit 124
fi
) &
watchdog_pid=$!
if [[ -n "${TASK13_ACTIVE_GROUP:-}" && "$monitor_enabled" -eq 1 ]]; then
set -m
fi
if wait "$command_pid"; then
rc=0
else
rc=$?
fi
if [[ -s "$timeout_marker" ]]; then
wait "$watchdog_pid" 2>/dev/null || watchdog_rc=$?
else
kill "$watchdog_pid" 2>/dev/null || true
wait "$watchdog_pid" 2>/dev/null || true
fi
rm -f "$timeout_marker"
[[ "$watchdog_rc" -eq 124 ]] && return 124
return "$rc"
}
task13_supervised_call() {
TASK13_ACTIVE_GROUP="$BASHPID"
"$@"
}
task13_supervise() {
local seconds="$1" label="$2"
shift 2
task13_bounded "$seconds" "$label" task13_supervised_call "$@"
}
task13_compose_files() {
TASK13_COMPOSE=(
docker compose
--project-name "$TASK13_PROJECT"
--project-directory "$TASK13_ROOT"
--env-file "$TASK13_ENV_FILE"
-f "$TASK13_ROOT/compose.yaml"
)
if [[ "${TASK13_PROFILE:-local}" == server ]]; then
TASK13_COMPOSE+=(
-f "$TASK13_ROOT/deploy/compose.server.yaml"
-f "$TASK13_ROOT/deploy/compose.session-server.yaml.example"
-f "$TASK13_OVERRIDE"
)
else
TASK13_COMPOSE+=(
-f "$TASK13_ROOT/deploy/compose.local.yaml"
-f "$TASK13_OVERRIDE"
)
fi
if [[ "${TASK13_PROFILE:-local}" == local && -f "$TASK13_CURRENT_IMAGE_OVERRIDE" ]]; then
TASK13_COMPOSE+=(-f "$TASK13_CURRENT_IMAGE_OVERRIDE")
fi
}
task13_compose() {
task13_compose_files
"${TASK13_COMPOSE[@]}" "$@"
}
task13_compose_logged() {
local label="$1"
shift
task13_compose_files
task13_run_logged "$label" "${TASK13_COMPOSE[@]}" "$@"
}
task13_report_core_startup_failure() {
local container_id state exit_code logs cause="startup failure is unclassified"
container_id="$(task13_compose ps --all -q core 2>/dev/null | head -n 1 || true)"
state=""
if [[ -n "$container_id" ]]; then
state="$(docker inspect --format '{{.State.Status}}:{{.State.ExitCode}}' "$container_id" 2>/dev/null || true)"
fi
exit_code="${state##*:}"
[[ "$exit_code" =~ ^[0-9]{1,3}$ ]] || exit_code="unknown"
logs="$(task13_compose logs --no-color --tail 100 core 2>/dev/null || true)"
case "$logs" in
*auth_session_store_invalid*|*auth*storage*request*failed*|*EACCES*auth*|*permission*auth*)
cause="authentication state storage is unavailable"
;;
*auth_config_invalid*|*authentication*configuration*)
cause="authentication configuration is invalid"
;;
*workspace_registry_invalid*|*workspace*registry*)
cause="workspace registry startup validation failed"
;;
esac
printf 'Core startup cause: %s (exit code %s).\n' "$cause" "$exit_code" >&2
}
task13_compose_start_logged() {
local label="$1"
shift
task13_compose_files
if task13_bounded "$TASK13_COMMAND_TIMEOUT" "$label" \
"${TASK13_COMPOSE[@]}" "$@" >>"$TASK13_LOG" 2>&1; then
return 0
fi
TASK13_FAILURE_LOGGED=1
printf 'Task 13 command failed: %s\n' "$label" >&2
task13_report_core_startup_failure
return 1
}
task13_write_environment() {
local remote="$1"
{
printf 'THOTH_HTTP_PORT=%s\n' "$TASK13_FRONTEND_PORT"
printf 'THOTH_CORE_HTTP_PORT=0\n'
printf 'MAX_PI_PROCESSES=2\n'
printf 'PI_AUTH_FILE=%s\n' "$TASK13_PI_AUTH"
printf 'THT_SECRETS_FILE=%s\n' "$TASK13_SECRETS"
printf 'THT_AUTH_CONFIG_ROOT=%s\n' "$TASK13_AUTH_ROOT"
printf 'THT_WORKSPACE_GIT_REMOTE=%s\n' "$remote"
printf 'THT_WORKSPACE_GIT_BRANCH=%s\n' "$TASK13_BRANCH"
printf 'THT_LLM_URL=http://%s:9000/v1\n' "$TASK13_LLM_CONTAINER"
} >"$TASK13_ENV_FILE"
chmod 0600 "$TASK13_ENV_FILE"
}
task13_write_fixture_files() {
printf '{}\n' >"$TASK13_PI_AUTH"
printf 'THT_MODEL_API_KEY=%s\n' "$TASK13_SECRET_VALUE" >"$TASK13_SECRETS"
printf '%s' "task13-runtime-password-$TASK13_RUN_ID" >"$TASK13_SESSION_RUNTIME_PASSWORD"
printf '%s' "$TASK13_AUTH_PASSWORD" >"$TASK13_AUTH_PASSWORD_FILE"
mkdir -p "$TASK13_AUTH_ROOT"
chmod 0644 "$TASK13_PI_AUTH"
chmod 0700 "$TASK13_AUTH_ROOT"
chmod 0600 "$TASK13_SECRETS" "$TASK13_SESSION_RUNTIME_PASSWORD" "$TASK13_AUTH_PASSWORD_FILE"
cat >"$TASK13_PI_MODELS" <<EOF
{
"providers": {
"local-qwen": {
"baseUrl": "http://$TASK13_LLM_CONTAINER:9000/v1",
"api": "openai-completions",
"apiKey": "task13-local-provider",
"models": [
{
"id": "task13-smoke",
"name": "Task 13 deterministic smoke",
"reasoning": false,
"contextWindow": 4096,
"maxTokens": 64
}
]
}
}
}
EOF
cat >"$TASK13_PI_SETTINGS" <<'EOF'
{
"defaultProjectTrust": "always",
"enabledModels": ["local-qwen/task13-smoke"]
}
EOF
chmod 0644 "$TASK13_PI_MODELS" "$TASK13_PI_SETTINGS"
cat >"$TASK13_LLM_SERVER" <<'EOF'
import http from "node:http";
const server = http.createServer((request, response) => {
if (request.method === "GET" && request.url === "/health") {
response.writeHead(200, { "content-type": "application/json" });
response.end('{"status":"ok"}');
return;
}
if (request.method === "GET" && request.url === "/v1/models") {
response.writeHead(200, { "content-type": "application/json" });
response.end('{"object":"list","data":[{"id":"task13-smoke","object":"model"}]}');
return;
}
if (request.method !== "POST" || request.url !== "/v1/chat/completions") {
response.writeHead(404, { "content-type": "application/json" });
response.end('{"error":{"message":"not found"}}');
return;
}
let body = "";
request.setEncoding("utf8");
request.on("data", (chunk) => { body += chunk; });
request.on("end", () => {
let stream = true;
try { stream = JSON.parse(body).stream !== false; } catch { /* return the safe fixture */ }
if (!stream) {
response.writeHead(200, { "content-type": "application/json" });
response.end(JSON.stringify({
id: "task13", object: "chat.completion", created: 1, model: "task13-smoke",
choices: [{ index: 0, message: { role: "assistant", content: "OK" }, finish_reason: "stop" }],
}));
return;
}
response.writeHead(200, {
"content-type": "text/event-stream",
"cache-control": "no-cache",
connection: "keep-alive",
});
response.write('data: {"id":"task13","object":"chat.completion.chunk","created":1,"model":"task13-smoke","choices":[{"index":0,"delta":{"role":"assistant","content":"OK"},"finish_reason":null}]}\n\n');
response.write('data: {"id":"task13","object":"chat.completion.chunk","created":1,"model":"task13-smoke","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}\n\n');
response.end("data: [DONE]\n\n");
});
});
server.listen(9000, "0.0.0.0");
EOF
chmod 0644 "$TASK13_LLM_SERVER"
cat >"$TASK13_OVERRIDE" <<EOF
services:
core:
image: $TASK13_CORE_IMAGE
build:
labels:
io.thothii.task13.run: "$TASK13_RUN_ID"
environment:
PI_PROVIDER: local-qwen
PI_MODEL: task13-smoke
PI_THINKING: low
THT_WORKSPACE_INSTALLATION_ID: task13-smoke
THT_WORKSPACE_REGISTRY_ROOT: /data/workspace-registry
THT_WS_TASK13_SMOKE_DWH_TRANSPORT: postgres_direct
THT_WS_TASK13_SMOKE_DWH_HOST: dwh.task13.invalid
THT_WS_TASK13_SMOKE_DWH_PORT: "5432"
THT_WS_TASK13_SMOKE_DWH_USER: task13_reader
THT_WS_TASK13_SMOKE_DWH_PASSWORD_FILE: /run/secrets/task13-runtime-password
labels:
io.thothii.task13.run: "$TASK13_RUN_ID"
volumes: !override
- settings:/data/settings
- pi-state:/home/thoth/.pi
- $TASK13_PI_AUTH:/home/thoth/.pi/agent/auth.json:ro
- $TASK13_PI_MODELS:/home/thoth/.pi/agent/models.json:ro
- $TASK13_PI_SETTINGS:/home/thoth/.pi/agent/settings.json:ro
- workspace-registry:/data/workspace-registry
- workspace-secrets:/data/workspace-secrets
- sessions:/data/sessions
- $TASK13_AUTH_ROOT:/run/thothii-auth:ro
- auth-state:/data/auth
- $TASK13_SESSION_RUNTIME_PASSWORD:/run/secrets/task13-runtime-password:ro
- $TASK13_REMOTE:/fixtures/remote.git:ro
frontend:
image: $TASK13_FRONTEND_IMAGE
build:
labels:
io.thothii.task13.run: "$TASK13_RUN_ID"
labels:
io.thothii.task13.run: "$TASK13_RUN_ID"
qdrant:
# The normal local profile exposes a developer dashboard; the isolated smoke needs no host port.
ports: !reset []
labels:
io.thothii.task13.run: "$TASK13_RUN_ID"
embedding:
labels:
io.thothii.task13.run: "$TASK13_RUN_ID"
embedding-model-init:
labels:
io.thothii.task13.run: "$TASK13_RUN_ID"
networks:
thothii:
labels:
io.thothii.task13.run: "$TASK13_RUN_ID"
volumes:
settings:
labels:
io.thothii.task13.run: "$TASK13_RUN_ID"
pi-state:
labels:
io.thothii.task13.run: "$TASK13_RUN_ID"
workspace-registry:
labels:
io.thothii.task13.run: "$TASK13_RUN_ID"
workspace-secrets:
labels:
io.thothii.task13.run: "$TASK13_RUN_ID"
sessions:
labels:
io.thothii.task13.run: "$TASK13_RUN_ID"
auth-state:
labels:
io.thothii.task13.run: "$TASK13_RUN_ID"
qdrant-data:
labels:
io.thothii.task13.run: "$TASK13_RUN_ID"
embedding-models:
labels:
io.thothii.task13.run: "$TASK13_RUN_ID"
EOF
chmod 0600 "$TASK13_OVERRIDE"
cat >"$TASK13_INSTALLATION" <<EOF
profile: local
projectDirectory: "$TASK13_ROOT"
envFile: "$TASK13_ENV_FILE"
authentication:
configDirectory: "$TASK13_AUTH_ROOT"
overrides:
- "$TASK13_OVERRIDE"
EOF
chmod 0600 "$TASK13_INSTALLATION"
}
task13_write_server_fixture_files() {
local data_root pi_root registry_root remote_path workspace_path
TASK13_OIDC_SERVER="${TASK13_OIDC_SERVER:-$TASK13_TMP/fake-oidc.mjs}"
TASK13_OIDC_CERT="${TASK13_OIDC_CERT:-$TASK13_TMP/fake-oidc-cert.pem}"
TASK13_OIDC_KEY="${TASK13_OIDC_KEY:-$TASK13_TMP/fake-oidc-key.pem}"
printf '{}\n' >"$TASK13_PI_AUTH"
printf 'THT_MODEL_API_KEY=%s\nTHT_OIDC_CLIENT_SECRET=%s\nTHT_AUTHENTIK_API_TOKEN=%s\n' \
"$TASK13_SECRET_VALUE" "$TASK13_OIDC_CLIENT_SECRET" "$TASK13_AUTHENTIK_API_TOKEN" >"$TASK13_SECRETS"
printf '%s' "$TASK13_SESSION_PASSWORD" >"$TASK13_SESSION_RUNTIME_PASSWORD"
printf '%s' "$TASK13_SESSION_MIGRATOR_PASSWORD" >"$TASK13_SESSION_MIGRATOR_PASSWORD_FILE"
cat >"$TASK13_SESSION_CA" <<'EOF'
-----BEGIN CERTIFICATE-----
VEFTSzEzLURJU1BPU0FCTEUtU0VTU0lPTi1DQQ==
-----END CERTIFICATE-----
EOF
chmod 0644 "$TASK13_PI_AUTH" "$TASK13_SESSION_CA"
chmod 0600 "$TASK13_SECRETS" "$TASK13_SESSION_RUNTIME_PASSWORD" \
"$TASK13_SESSION_MIGRATOR_PASSWORD_FILE"
task13_run_logged "create scoped fake OIDC certificate" openssl req -x509 -newkey rsa:2048 \
-sha256 -nodes -days 1 -subj '/CN=task13-fake-oidc' \
-addext 'subjectAltName=DNS:task13-fake-oidc' \
-keyout "$TASK13_OIDC_KEY" -out "$TASK13_OIDC_CERT"
chmod 0600 "$TASK13_OIDC_KEY"
chmod 0644 "$TASK13_OIDC_CERT"
cat >"$TASK13_OIDC_SERVER" <<'EOF'
import { generateKeyPairSync } from "node:crypto";
import { readFileSync } from "node:fs";
import https from "node:https";
const origin = "https://task13-fake-oidc:9443";
const issuer = `${origin}/application/o/task13/`;
const expectedToken = process.env.TASK13_AUTHENTIK_API_TOKEN;
const { publicKey } = generateKeyPairSync("rsa", { modulusLength: 2048 });
const jwk = { ...publicKey.export({ format: "jwk" }), kid: "task13", use: "sig", alg: "RS256" };
const send = (response, status, body) => {
const payload = JSON.stringify(body);
response.writeHead(status, { "content-type": "application/json", "content-length": Buffer.byteLength(payload) });
response.end(payload);
};
const server = https.createServer({
cert: readFileSync("/fixtures/oidc-cert.pem"),
key: readFileSync("/fixtures/oidc-key.pem"),
}, (request, response) => {
const target = new URL(request.url ?? "/", origin);
if (target.pathname === "/health") return send(response, 200, { status: "ok" });
if (target.pathname.includes(".well-known/openid-configuration")) {
return send(response, 200, {
issuer,
authorization_endpoint: `${origin}/authorize`,
token_endpoint: `${origin}/token`,
jwks_uri: `${origin}/jwks`,
response_types_supported: ["code"],
subject_types_supported: ["public"],
id_token_signing_alg_values_supported: ["RS256"],
});
}
if (target.pathname === "/jwks") return send(response, 200, { keys: [jwk] });
if (target.pathname === "/api/v3/core/groups/") {
if (request.headers.authorization !== `Bearer ${expectedToken}`) return send(response, 401, { detail: "unauthorized" });
const name = target.searchParams.get("name") ?? "";
console.log(`group:${name}`);
const configured = name === "task13-users" || name === "task13-admins";
return send(response, 200, {
pagination: { next: null },
results: configured ? [{ name }, { name: "task13-unrelated" }] : [{ name: "task13-unrelated" }],
});
}
return send(response, 404, { error: "not_found" });
});
server.listen(9443, "0.0.0.0");
EOF
chmod 0644 "$TASK13_OIDC_SERVER"
cat >"$TASK13_SERVER_WORKSPACE_CONFIG" <<'EOF'
language: en
session_storage:
type: postgres_direct
connection:
host: ${THT_SESSION_DB_HOST}
port: ${THT_SESSION_DB_PORT}
database: ${THT_SESSION_DB_NAME}
schema: thoth_sessions
user: ${THT_SESSION_RUNTIME_USER}
password_file: ${THT_SESSION_RUNTIME_PASSWORD_FILE}
sslmode: ${THT_SESSION_DB_SSLMODE}
sslrootcert: ${THT_SESSION_DB_SSLROOTCERT}
roots:
artifacts: artifacts
indexes: indexes
sessions: sessions
EOF
chmod 0600 "$TASK13_SERVER_WORKSPACE_CONFIG"
mkdir -p "$TASK13_SERVER_DATA" "$TASK13_SERVER_PI_STATE" "$TASK13_SERVER_REGISTRY"
mkdir -p "$TASK13_AUTH_ROOT"
"$TASK13_ROOT/scripts/prepare-server-pi-state.sh" \
"$TASK13_SERVER_PI_STATE" "$(id -u)" "$(id -g)" >>"$TASK13_LOG"
chmod 0777 "$TASK13_SERVER_DATA" "$TASK13_SERVER_PI_STATE" \
"$TASK13_SERVER_PI_STATE/agent" "$TASK13_SERVER_REGISTRY"
chmod 0700 "$TASK13_AUTH_ROOT"
data_root="$TASK13_SERVER_DATA"
pi_root="$TASK13_SERVER_PI_STATE"
registry_root="$TASK13_SERVER_REGISTRY"
remote_path="$TASK13_REMOTE"
workspace_path="$TASK13_SERVER_WORKSPACE_CONFIG"
cat >"$TASK13_OVERRIDE" <<EOF
services:
core:
image: $TASK13_CORE_IMAGE
build:
labels:
io.thothii.task13.run: "$TASK13_RUN_ID"
environment:
NODE_EXTRA_CA_CERTS: /fixtures/task13-oidc-ca.pem
THT_WS_TASK13_SMOKE_DWH_TRANSPORT: postgres_direct
THT_WS_TASK13_SMOKE_DWH_HOST: dwh.task13.invalid
THT_WS_TASK13_SMOKE_DWH_PORT: "5432"
THT_WS_TASK13_SMOKE_DWH_USER: task13_reader
THT_WS_TASK13_SMOKE_DWH_PASSWORD_FILE: /run/secrets/task13-runtime-password
labels:
io.thothii.task13.run: "$TASK13_RUN_ID"
volumes:
- $remote_path:/fixtures/remote.git:ro
- $TASK13_SESSION_RUNTIME_PASSWORD:/run/secrets/task13-runtime-password:ro
- $TASK13_OIDC_CERT:/fixtures/task13-oidc-ca.pem:ro
frontend:
image: $TASK13_FRONTEND_IMAGE
build:
labels:
io.thothii.task13.run: "$TASK13_RUN_ID"
labels:
io.thothii.task13.run: "$TASK13_RUN_ID"
qdrant:
labels:
io.thothii.task13.run: "$TASK13_RUN_ID"
embedding:
labels:
io.thothii.task13.run: "$TASK13_RUN_ID"
embedding-model-init:
labels:
io.thothii.task13.run: "$TASK13_RUN_ID"
session-migrate:
image: $TASK13_CORE_IMAGE
networks:
thothii:
labels:
io.thothii.task13.run: "$TASK13_RUN_ID"
volumes:
qdrant-data:
labels:
io.thothii.task13.run: "$TASK13_RUN_ID"
embedding-models:
labels:
io.thothii.task13.run: "$TASK13_RUN_ID"
EOF
chmod 0600 "$TASK13_OVERRIDE"
{
printf 'THOTH_HTTP_PORT=0\n'
printf 'THOTH_SERVER_BIND=127.0.0.1\n'
printf 'MAX_PI_PROCESSES=2\n'
printf 'PI_AUTH_FILE=%s\n' "$TASK13_PI_AUTH"
printf 'THT_SECRETS_FILE=%s\n' "$TASK13_SECRETS"
printf 'THT_AUTH_CONFIG_ROOT=%s\n' "$TASK13_AUTH_ROOT"
printf 'THT_WORKSPACE_GIT_REMOTE=/fixtures/remote.git\n'
printf 'THT_WORKSPACE_GIT_BRANCH=%s\n' "$TASK13_BRANCH"
printf 'THT_DATA_ROOT=%s\n' "$data_root"
printf 'THT_PI_STATE_ROOT=%s\n' "$pi_root"
printf 'THT_WORKSPACE_REGISTRY_ROOT=%s\n' "$registry_root"
printf 'THT_SERVER_WORKSPACE_CONFIG=%s\n' "$workspace_path"
printf 'THT_SESSION_DB_HOST=task13-session.invalid\n'
printf 'THT_SESSION_DB_PORT=5432\n'
printf 'THT_SESSION_DB_NAME=task13\n'
printf 'THT_SESSION_RUNTIME_USER=task13_runtime\n'
printf 'THT_SESSION_MIGRATOR_USER=task13_migrator\n'
printf 'THT_SESSION_DB_SSLMODE=verify-full\n'
printf 'THT_SESSION_RUNTIME_PASSWORD_SOURCE=%s\n' "$TASK13_SESSION_RUNTIME_PASSWORD"
printf 'THT_SESSION_MIGRATOR_PASSWORD_SOURCE=%s\n' "$TASK13_SESSION_MIGRATOR_PASSWORD_FILE"
printf 'THT_SESSION_CA_SOURCE=%s\n' "$TASK13_SESSION_CA"
printf 'THT_LLM_URL=https://llm.task13.invalid/v1\n'
} >"$TASK13_ENV_FILE"
chmod 0600 "$TASK13_ENV_FILE"
cat >"$TASK13_INSTALLATION" <<EOF
profile: server
projectDirectory: "$TASK13_ROOT"
envFile: "$TASK13_ENV_FILE"
authentication:
configDirectory: "$TASK13_AUTH_ROOT"
overrides:
- "$TASK13_ROOT/deploy/compose.session-server.yaml.example"
- "$TASK13_OVERRIDE"
EOF
chmod 0600 "$TASK13_INSTALLATION"
}
task13_workspace_metadata() {
local fixture="$1"
awk '
/^workspace:[[:space:]]*$/ { in_workspace = 1; next }
in_workspace && /^[^[:space:]]/ { in_workspace = 0 }
in_workspace && /^ id:[[:space:]]*/ {
sub(/^ id:[[:space:]]*/, "")
id = $0
next
}
in_workspace && /^ name:[[:space:]]*/ {
sub(/^ name:[[:space:]]*/, "")
name = $0
next
}
in_workspace && /^ description:[[:space:]]*/ {
sub(/^ description:[[:space:]]*/, "")
description = $0
next
}
END {
if (id == "" || name == "") exit 1
print id
print name
if (description != "") print description
}
' "$fixture"
}
task13_write_catalog() {
local catalog_path="$1" id="$2" name="$3" description="${4:-}"
{
printf 'schema_version: 1\n'
printf 'workspaces:\n'
printf ' - id: %s\n' "$id"
printf ' name: %s\n' "$name"
if [[ -n "$description" ]]; then
printf ' description: %s\n' "$description"
fi
} >"$catalog_path"
}
task13_replace_once() {
local target="$1" old="$2" new="$3"
python3 - "$target" "$old" "$new" <<'PY'
from pathlib import Path
import sys
path = Path(sys.argv[1])
old = sys.argv[2]
new = sys.argv[3]
text = path.read_text()
count = text.count(old)
if count != 1:
raise SystemExit(f"expected exactly one occurrence of {old!r} in {path}, found {count}")
path.write_text(text.replace(old, new, 1))
PY
}
task13_commit_registry_change() {
local message="$1"
task13_run_logged "$message" git -C "$TASK13_SEED" add -A
task13_run_logged "$message" git -C "$TASK13_SEED" -c user.name='Task 13 Smoke' -c user.email='task13-smoke@example.invalid' commit -m "$message"
task13_run_logged "$message" git -C "$TASK13_SEED" push "$TASK13_REMOTE" "HEAD:$TASK13_BRANCH"
chmod -R a+rX "$TASK13_REMOTE"
}
task13_seed_registry() {
local fixture metadata
fixture="$TASK13_ROOT/scripts/fixtures/workspace-registry-task13.yaml"
mapfile -t metadata < <(task13_workspace_metadata "$fixture")
TASK13_WORKSPACE_ID="${metadata[0]}"
TASK13_WORKSPACE_NAME="${metadata[1]}"
TASK13_WORKSPACE_DESCRIPTION="${metadata[2]-}"
mkdir -p "$TASK13_SEED"
task13_run_logged "initialize bare workspace registry" git init --bare --initial-branch=main "$TASK13_REMOTE"
task13_run_logged "initialize workspace seed" git -C "$TASK13_SEED" init --initial-branch=main
mkdir -p "$TASK13_SEED/$TASK13_WORKSPACE_ID"
cp "$fixture" "$TASK13_SEED/$TASK13_WORKSPACE_ID/workspace.yaml"
task13_write_catalog "$TASK13_SEED/thoth-workspaces.yaml" "$TASK13_WORKSPACE_ID" "$TASK13_WORKSPACE_NAME" "$TASK13_WORKSPACE_DESCRIPTION"
if grep -Fq 'type: filesystem' "$fixture"; then
mkdir -p "$TASK13_SEED/$TASK13_WORKSPACE_ID/evidence"
printf 'guide v1\n' >"$TASK13_SEED/$TASK13_WORKSPACE_ID/evidence/guide.md"
fi
task13_commit_registry_change 'Seed Task 13 workspace registry'
}
task13_build_tht() {
local os arch
mkdir -p "$TASK13_THT_DIR"
task13_run_logged "build tht cross-platform binaries" env \
THT_THT_OUTPUT_DIRECTORY="$TASK13_THT_DIR" \
bash "$TASK13_ROOT/scripts/build-tht.sh"
os="$(uname -s)"
arch="$(uname -m)"
case "$os/$arch" in
Darwin/x86_64) TASK13_THT="$TASK13_THT_DIR/tht-darwin-amd64" ;;
Darwin/arm64) TASK13_THT="$TASK13_THT_DIR/tht-darwin-arm64" ;;
Linux/x86_64|Linux/amd64) TASK13_THT="$TASK13_THT_DIR/tht-linux-amd64" ;;
Linux/aarch64|Linux/arm64) TASK13_THT="$TASK13_THT_DIR/tht-linux-arm64" ;;
*) task13_fail "unsupported smoke host: $os/$arch" ;;
esac
chmod 0700 "$TASK13_THT"
task13_run_logged "invoke host tht" "$TASK13_THT" --help
}
task13_assert_rendered_contract() {
local services rendered
services="$(task13_compose config --services | sort)"
[[ "$services" == $'core\nembedding\nembedding-model-init\nfrontend\nqdrant' ]] \
|| task13_fail "rendered stack is not the mandatory internal semantic topology"
rendered="$TASK13_TMP/rendered-compose.yaml"
task13_compose config >"$rendered"
if grep -Eqi 'docker\.sock|/var/run/docker' "$rendered"; then
task13_fail "rendered Compose exposes a Docker daemon endpoint"
fi
if grep -Fq "$TASK13_SECRET_VALUE" "$rendered"; then
task13_fail "rendered Compose exposed the fixture secret"
fi
for endpoint in THT_DWH_REST_URL THT_LLM_URL \
THT_INTERNAL_QDRANT_URL THT_INTERNAL_EMBEDDING_URL \
THT_INTERNAL_EMBEDDING_MODEL THT_INTERNAL_EMBEDDING_DIMENSIONS; do
grep -Fq "$endpoint" "$rendered" || task13_fail "rendered Compose lacks $endpoint"
done
if grep -Eq 'THT_VEC_REST_URL|THT_VEC_WRITE_REST_URL|THT_OLLAMA_URL' "$rendered"; then
task13_fail "rendered Compose still exposes retired external semantic bindings"
fi
grep -Fq '/run/thothii-auth' "$rendered" || task13_fail "rendered Compose lacks the read-only auth configuration mount"
grep -Fq '/data/auth' "$rendered" || task13_fail "rendered Compose lacks authentication state storage"
}
task13_configure_local_authentication() {
task13_run_logged "configure local authentication" "$TASK13_THT" --installation "$TASK13_INSTALLATION" auth configure \
--mode local --public-url "http://127.0.0.1:$TASK13_FRONTEND_PORT" \
--admin-user "$TASK13_AUTH_ADMIN" --admin-display-name "Task 13 Administrator" \
--password-file "$TASK13_AUTH_PASSWORD_FILE"
}
task13_configure_server_oidc_authentication() {
task13_run_logged "configure fake server OIDC authentication" "$TASK13_THT" --installation "$TASK13_INSTALLATION" auth configure \
--mode oidc --public-url "https://task13.example.invalid" \
--issuer "https://task13-fake-oidc:9443/application/o/task13/" --client-id task13-smoke-client \
--authentik-base-url "https://task13-fake-oidc:9443" --user-group task13-users --admin-group task13-admins
}
task13_start_stack() {
printf '== Build and start isolated local Compose distribution ==\n'
task13_assert_rendered_contract
task13_compose_logged "build local Compose images" build --pull
task13_compose_start_logged "start local Compose distribution" up --detach --wait --wait-timeout 120
TASK13_NETWORK="$(docker network ls \
--filter "label=com.docker.compose.project=$TASK13_PROJECT" \
--filter 'label=com.docker.compose.network=thothii' --format '{{.Name}}')"
[[ -n "$TASK13_NETWORK" && "$TASK13_NETWORK" != *$'\n'* ]] || task13_fail "isolated Compose network was not resolved"
task13_run_logged "start deterministic local LLM fixture" docker run --detach \
--name "$TASK13_LLM_CONTAINER" \
--label "io.thothii.task13.run=$TASK13_RUN_ID" \
--network "$TASK13_NETWORK" \
--entrypoint node \
--volume "$TASK13_LLM_SERVER:/fixtures/fake-llm.mjs:ro" \
"$TASK13_CORE_IMAGE" /fixtures/fake-llm.mjs
for _attempt in $(seq 1 30); do
if docker exec "$TASK13_LLM_CONTAINER" node -e \
"fetch('http://127.0.0.1:9000/health',{signal:AbortSignal.timeout(3000)}).then(r=>{if(!r.ok)process.exit(1)}).catch(()=>process.exit(1))" \
>>"$TASK13_LOG" 2>&1; then
return 0
fi
sleep 1
done
task13_log_failure "deterministic local LLM fixture readiness"
}
task13_start_server_stack() {
printf '== Build and start isolated Linux server profile ==\n'
task13_assert_rendered_contract
task13_compose_logged "build server Compose images" build --pull core frontend
task13_compose_logged "create server Compose resources" create core frontend
TASK13_NETWORK="$(docker network ls \
--filter "label=com.docker.compose.project=$TASK13_PROJECT" \
--filter 'label=com.docker.compose.network=thothii' --format '{{.Name}}')"
[[ -n "$TASK13_NETWORK" && "$TASK13_NETWORK" != *$'\n'* ]] \
|| task13_fail "isolated server Compose network was not resolved"
task13_run_logged "start scoped fake OIDC provider" docker run --detach \
--name "$TASK13_OIDC_CONTAINER" \
--label "io.thothii.task13.run=$TASK13_RUN_ID" \
--network "$TASK13_NETWORK" --network-alias task13-fake-oidc \
--user "$(id -u):$(id -g)" \
--env "TASK13_AUTHENTIK_API_TOKEN=$TASK13_AUTHENTIK_API_TOKEN" \
--env NODE_EXTRA_CA_CERTS=/fixtures/oidc-cert.pem \
--entrypoint node \
--volume "$TASK13_OIDC_SERVER:/fixtures/fake-oidc.mjs:ro" \
--volume "$TASK13_OIDC_CERT:/fixtures/oidc-cert.pem:ro" \
--volume "$TASK13_OIDC_KEY:/fixtures/oidc-key.pem:ro" \
"$TASK13_CORE_IMAGE" /fixtures/fake-oidc.mjs
for _attempt in $(seq 1 30); do
if docker exec "$TASK13_OIDC_CONTAINER" node -e \
"fetch('https://task13-fake-oidc:9443/health',{signal:AbortSignal.timeout(3000)}).then(r=>{if(!r.ok)process.exit(1)}).catch(()=>process.exit(1))" \
>>"$TASK13_LOG" 2>&1; then
break
fi
sleep 1
done
docker exec "$TASK13_OIDC_CONTAINER" node -e \
"fetch('https://task13-fake-oidc:9443/health',{signal:AbortSignal.timeout(3000)}).then(r=>{if(!r.ok)process.exit(1)}).catch(()=>process.exit(1))" \
>>"$TASK13_LOG" 2>&1 || task13_log_failure "scoped fake OIDC provider readiness"
task13_compose_start_logged "start server Compose distribution" \
up --detach --wait --wait-timeout 120 core frontend
}
task13_frontend_address() {
task13_compose port frontend 8080 | awk 'NR == 1 {print $0}'
}
task13_core_id() {
task13_compose ps -q core
}
task13_assert_maintenance_auth_isolation() {
task13_compose_logged "workspace maintenance auth isolation" \
--profile workspace-maintenance run --rm --no-deps --entrypoint sh workspace-maintenance -ceu \
'test ! -e /run/thothii-auth && test ! -e /data/auth'
}
task13_assert_local_auth_lifecycle() {
local frontend cookie_jar login_body me csrf unauthenticated
frontend="$(task13_frontend_address)"
cookie_jar="$TASK13_TMP/local-auth.cookies"
login_body="$TASK13_TMP/local-auth-login.json"
printf '{"username":"%s","password":"%s","remember":true}' \
"$TASK13_AUTH_ADMIN" "$TASK13_AUTH_PASSWORD" >"$login_body"
chmod 0600 "$cookie_jar" "$login_body" 2>/dev/null || chmod 0600 "$login_body"
task13_run_logged "local auth configuration" curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" \
--max-time "$TASK13_CURL_MAX_TIME" --fail --silent --show-error "http://$frontend/api/auth/config"
task13_run_logged "local auth login" curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" \
--max-time "$TASK13_CURL_MAX_TIME" --fail --silent --show-error --cookie-jar "$cookie_jar" \
-H "Origin: http://$frontend" -H 'content-type: application/json' --data-binary "@$login_body" \
"http://$frontend/api/auth/local/login"
me="$(curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" --max-time "$TASK13_CURL_MAX_TIME" \
--fail --silent --show-error --cookie "$cookie_jar" "http://$frontend/api/me")"
node -e 'const value=JSON.parse(process.argv[1]); if(value.issuer!=="local"||value.session?.remembered!==true||typeof value.csrfToken!=="string") process.exit(1)' "$me" \
|| task13_fail "local login did not create a remembered authenticated session"
csrf="$(node -e 'process.stdout.write(JSON.parse(process.argv[1]).csrfToken)' "$me")"
task13_compose_logged "remembered local auth core restart" up --detach --force-recreate --wait --wait-timeout 120 core
me="$(curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" --max-time "$TASK13_CURL_MAX_TIME" \
--fail --silent --show-error --cookie "$cookie_jar" "http://$frontend/api/me")"
node -e 'const value=JSON.parse(process.argv[1]); if(value.session?.remembered!==true) process.exit(1)' "$me" \
|| task13_fail "remembered local session did not survive a core restart"
csrf="$(node -e 'process.stdout.write(JSON.parse(process.argv[1]).csrfToken)' "$me")"
task13_run_logged "local auth Pi management" curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" \
--max-time 45 --fail --silent --show-error --cookie "$cookie_jar" \
-H "Origin: http://$frontend" -H "x-thothii-csrf: $csrf" -X POST "http://$frontend/api/pi-management/test"
task13_run_logged "local auth logout" curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" \
--max-time "$TASK13_CURL_MAX_TIME" --fail --silent --show-error --cookie "$cookie_jar" \
-H "Origin: http://$frontend" -H "x-thothii-csrf: $csrf" -X POST "http://$frontend/api/auth/logout"
unauthenticated="$(curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" --max-time "$TASK13_CURL_MAX_TIME" \
--silent --output /dev/null --write-out '%{http_code}' --cookie "$cookie_jar" "http://$frontend/api/me")"
[[ "$unauthenticated" == 401 ]] || task13_fail "local logout did not revoke the remembered session"
}
task13_create_admin_session() {
local frontend login_body me
frontend="$(task13_frontend_address)"
TASK13_ADMIN_COOKIE="$TASK13_TMP/operations-admin.cookies"
login_body="$TASK13_TMP/operations-admin-login.json"
printf '{"username":"%s","password":"%s","remember":true}' \
"$TASK13_AUTH_ADMIN" "$TASK13_AUTH_PASSWORD" >"$login_body"
chmod 0600 "$login_body"
task13_run_logged "create authenticated smoke administration session" curl \
--connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" --max-time "$TASK13_CURL_MAX_TIME" \
--fail --silent --show-error --cookie-jar "$TASK13_ADMIN_COOKIE" \
-H "Origin: http://$frontend" -H 'content-type: application/json' --data-binary "@$login_body" \
"http://$frontend/api/auth/local/login"
me="$(curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" --max-time "$TASK13_CURL_MAX_TIME" \
--fail --silent --show-error --cookie "$TASK13_ADMIN_COOKIE" "http://$frontend/api/me")"
TASK13_ADMIN_CSRF="$(node -e 'const value=JSON.parse(process.argv[1]); if(typeof value.csrfToken!=="string") process.exit(1); process.stdout.write(value.csrfToken)' "$me")" \
|| task13_fail "authenticated smoke administration session lacks CSRF state"
}
task13_authenticated_get() {
local path="$1" frontend
frontend="$(task13_frontend_address)"
curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" --max-time "$TASK13_CURL_MAX_TIME" \
--fail --silent --show-error --cookie "$TASK13_ADMIN_COOKIE" "http://$frontend/api/$path"
}
task13_authenticated_post() {
local path="$1" frontend
frontend="$(task13_frontend_address)"
curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" --max-time 15 \
--fail --silent --show-error --cookie "$TASK13_ADMIN_COOKIE" \
-H "Origin: http://$frontend" -H "x-thothii-csrf: $TASK13_ADMIN_CSRF" \
-X POST "http://$frontend/api/$path"
}
task13_assert_local_restore_reauthentication() {
local frontend archive login_body cookie_before cookie_after me status_before status_after
frontend="$(task13_frontend_address)"
archive="$TASK13_TMP/local-restore-source.zip"
login_body="$TASK13_TMP/local-restore-login.json"
cookie_before="$TASK13_TMP/local-restore-before.cookies"
cookie_after="$TASK13_TMP/local-restore-after.cookies"
printf '{"username":"%s","password":"%s","remember":true}' \
"$TASK13_AUTH_ADMIN" "$TASK13_AUTH_PASSWORD" >"$login_body"
chmod 0600 "$login_body"
task13_run_logged "create pre-backup browser session" curl \
--connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" --max-time "$TASK13_CURL_MAX_TIME" \
--fail --silent --show-error --cookie-jar "$cookie_before" \
-H "Origin: http://$frontend" -H 'content-type: application/json' --data-binary "@$login_body" \
"http://$frontend/api/auth/local/login"
task13_compose_logged "stop local stack for backup" stop
task13_run_logged "create real default-custody backup" "$TASK13_THT" \
--installation "$TASK13_INSTALLATION" backup --output "$archive"
python3 - "$archive" <<'PY'
import json
import sys
import zipfile
with zipfile.ZipFile(sys.argv[1]) as archive:
manifest = json.loads(archive.read("manifest.json"))
volumes = [item["logical_name"] for item in manifest["volumes"]]
if volumes != ["embedding-models", "pi-state", "qdrant-data", "sessions", "settings", "workspace-registry", "workspace-secrets"]:
raise SystemExit(f"unexpected backup volume custody: {volumes}")
entries = manifest["entries"]
if any(item.get("logical_name") == "auth-state" or "/data/auth" in item.get("source_path", "") for item in entries):
raise SystemExit("default backup contains authentication runtime state")
auth = [item for item in entries if item["path"].startswith("authentication-secrets/")]
if len(auth) != 1 or not auth[0]["path"].endswith("-auth.yaml") or auth[0]["archived"]:
raise SystemExit("default backup auth custody is not an auth.yaml reference only")
if any(item["path"].endswith("users.yaml") for item in entries):
raise SystemExit("default backup contains users.yaml")
PY
task13_compose_start_logged "restart local stack before restore" up --detach --wait --wait-timeout 120
status_before="$(curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" --max-time "$TASK13_CURL_MAX_TIME" \
--silent --output /dev/null --write-out '%{http_code}' --cookie "$cookie_before" "http://$frontend/api/me")"
[[ "$status_before" == 200 ]] || task13_fail "pre-backup browser session did not survive an ordinary stop/start"
task13_run_logged "create post-backup browser session" curl \
--connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" --max-time "$TASK13_CURL_MAX_TIME" \
--fail --silent --show-error --cookie-jar "$cookie_after" \
-H "Origin: http://$frontend" -H 'content-type: application/json' --data-binary "@$login_body" \
"http://$frontend/api/auth/local/login"
me="$(curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" --max-time "$TASK13_CURL_MAX_TIME" \
--fail --silent --show-error --cookie "$cookie_after" "http://$frontend/api/me")"
node -e 'const value=JSON.parse(process.argv[1]); if(value.issuer!=="local"||value.session?.remembered!==true) process.exit(1)' "$me" \
|| task13_fail "post-backup browser session was not authenticated"
task13_compose_logged "seed pending OIDC state excluded from restore" exec -T core sh -ceu \
'printf %s "{}" > /data/auth/oidc/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.json && chmod 0600 /data/auth/oidc/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.json && test "$(find /data/auth/sessions -type f | wc -l | tr -d " ")" -ge 2'
task13_compose_logged "stop local stack for restore" stop
task13_run_logged "perform real production restore" "$TASK13_THT" \
--installation "$TASK13_INSTALLATION" restore "$archive" --yes
task13_compose_start_logged "start restored local stack" up --detach --wait --wait-timeout 120
status_before="$(curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" --max-time "$TASK13_CURL_MAX_TIME" \
--silent --output /dev/null --write-out '%{http_code}' --cookie "$cookie_before" "http://$frontend/api/me")"
status_after="$(curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" --max-time "$TASK13_CURL_MAX_TIME" \
--silent --output /dev/null --write-out '%{http_code}' --cookie "$cookie_after" "http://$frontend/api/me")"
[[ "$status_before" == 401 && "$status_after" == 401 ]] \
|| task13_fail "restore did not force every independent browser session to reauthenticate"
task13_compose_logged "verify private empty restored auth state" exec -T core sh -ceu '
test "$(stat -c %a /data/auth)" = 700
test "$(stat -c %a /data/auth/sessions)" = 700
test "$(stat -c %a /data/auth/oidc)" = 700
test "$(stat -c %u /data/auth)" = "$(id -u)"
test -z "$(find /data/auth/sessions /data/auth/oidc -mindepth 1 -print -quit)"
'
task13_create_admin_session
}
task13_assert_runtime() {
local frontend expected_pi actual_pi core_id
frontend="$(task13_frontend_address)"
expected_pi="$(sed -n 's/^ARG PI_VERSION=//p' "$TASK13_ROOT/docker/core.Dockerfile" | head -n 1)"
actual_pi="$(task13_compose exec -T core pi --version | tr -d '\r\n')"
[[ -n "$expected_pi" && "$actual_pi" == "$expected_pi" ]] || task13_fail "embedded Pi version mismatch"
task13_run_logged "frontend health" curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" \
--max-time "$TASK13_CURL_MAX_TIME" --fail --silent --show-error "http://$frontend/"
task13_run_logged "same-origin core health" curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" \
--max-time "$TASK13_CURL_MAX_TIME" --fail --silent --show-error "http://$frontend/api/health"
task13_compose_logged "core non-root identity" exec -T core sh -ceu \
'test "$(id -u)" = 10001'
task13_compose_logged "embedded Pi executable" exec -T core sh -ceu \
'command -v pi >/dev/null'
task13_compose_logged "core Docker socket isolation" exec -T core sh -ceu \
'test ! -e /var/run/docker.sock'
task13_assert_local_auth_lifecycle
task13_create_admin_session
task13_authenticated_get workspace-registry/status >/dev/null \
|| task13_fail "authenticated workspace registry bootstrap failed"
task13_compose_logged "active workspace registry state" exec -T core sh -ceu \
'test -f /data/workspace-registry/state/active.json'
task13_compose_logged "mounted Pi auth readability" exec -T core sh -ceu \
'test -r /home/thoth/.pi/agent/auth.json'
task13_compose_logged "mounted application secret readability" exec -T core sh -ceu \
'test -r /run/secrets/thothii.secrets'
core_id="$(task13_core_id)"
[[ "$(docker inspect --format '{{ index .Config.Labels "io.thothii.task13.run" }}' "$core_id")" == "$TASK13_RUN_ID" ]] \
|| task13_fail "core lacks the explicit Task 13 resource label"
task13_assert_maintenance_auth_isolation
task13_run_logged "tht Pi doctor" "$TASK13_THT" --installation "$TASK13_INSTALLATION" pi doctor
}
task13_report_server_workspace_failure() {
local status="$1" response="$2"
printf 'authenticated server /api/workspaces returned HTTP %s\n' "$status" >&2
printf '%s\n' '--- sanitized server workspace response ---' >&2
if [[ -s "$response" ]]; then
tail -c 16384 "$response" | task13_sanitize >&2
else
printf '%s\n' '(empty response)' >&2
fi
printf '%s\n' '--- sanitized registry integrity probe ---' >&2
{
task13_compose exec -T core node --input-type=module -e '
const { loadConfig } = await import("/app/backend/dist/config.js");
const { WorkspaceRegistry } = await import("/app/backend/dist/workspaces/registry.js");
const registry = new WorkspaceRegistry(loadConfig(process.env).workspaceRegistry);
try {
const revisions = await registry.list();
for (const revision of revisions) await registry.read(revision.id);
console.log(JSON.stringify({ ok: true, revisions: revisions.length }));
} catch (error) {
console.log(JSON.stringify({
ok: false,
name: error instanceof Error ? error.name : "UnknownError",
code: error && typeof error === "object" && "code" in error ? error.code : "unknown",
message: error instanceof Error ? error.message : "Unknown registry failure",
}));
process.exitCode = 1;
}
' 2>&1 || printf '%s\n' '(registry integrity probe unavailable)'
} | tail -n 20 | task13_sanitize >&2
printf '%s\n' '--- sanitized core logs (last 100 lines) ---' >&2
{
task13_compose logs --no-color --tail 100 core 2>&1 \
|| printf '%s\n' '(core logs unavailable)'
} | tail -n 100 | task13_sanitize >&2
}
task13_assert_server_runtime() {
local frontend unauthenticated trusted_header_status session_status diagnostics status provider_requests core_id frontend_id
local expected_core_image expected_frontend_image
frontend="$(task13_frontend_address)"
task13_run_logged "server frontend health" curl \
--connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" --max-time "$TASK13_CURL_MAX_TIME" \
--fail --silent --show-error "http://$frontend/"
task13_run_logged "server same-origin core health" curl \
--connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" --max-time "$TASK13_CURL_MAX_TIME" \
--fail --silent --show-error "http://$frontend/api/health"
core_id="$(task13_core_id)"
frontend_id="$(task13_compose ps -q frontend)"
expected_core_image="$(docker image inspect --format '{{.Id}}' "$TASK13_CORE_IMAGE")"
expected_frontend_image="$(docker image inspect --format '{{.Id}}' "$TASK13_FRONTEND_IMAGE")"
[[ "$(docker inspect --format '{{.Image}}' "$core_id")" == "$expected_core_image" ]] \
|| task13_fail "server core did not use the smoke-built core image"
[[ "$(docker inspect --format '{{.Image}}' "$frontend_id")" == "$expected_frontend_image" ]] \
|| task13_fail "server frontend did not use the smoke-built frontend image"
task13_compose exec -T core sh -ceu '
test -r /run/thothii-auth/auth.yaml
test -d /data/auth
test -z "${AUTH_MODE+x}"
test "$THT_SESSION_STORAGE" = postgres
test -r /run/secrets/thothii.secrets
test -r /run/secrets/session_runtime_password
test -r /run/secrets/session_ca.pem
test -r /app/harness/workspaces/server-sessions.yaml
'
task13_mount_fingerprint | grep -Fq '/data = bind :' \
|| task13_fail "server profile did not bind the disposable data root"
task13_mount_fingerprint | grep -Fq '/home/thoth/.pi = bind :' \
|| task13_fail "server profile did not bind the disposable Pi state root"
task13_mount_fingerprint | grep -Fq '/data/workspace-registry = bind :' \
|| task13_fail "server profile did not bind the disposable registry root"
task13_assert_maintenance_auth_isolation
unauthenticated="$(curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" \
--max-time "$TASK13_CURL_MAX_TIME" --silent --output /dev/null --write-out '%{http_code}' \
"http://$frontend/api/workspaces")"
[[ "$unauthenticated" == 401 ]] || task13_fail "server profile did not enforce OIDC authentication"
trusted_header_status="$(curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" \
--max-time "$TASK13_CURL_MAX_TIME" --silent --output /dev/null --write-out '%{http_code}' \
-H 'x-thoth-trusted-principal-issuer: task13-proxy' \
-H 'x-thoth-trusted-principal-subject: task13-user' \
-H 'x-thoth-trusted-principal-display-name: Task 13 User' \
-H 'x-thoth-trusted-is-admin: 0' \
"http://$frontend/api/workspaces")"
[[ "$trusted_header_status" == 401 ]] || task13_fail "server accepted retired trusted identity headers"
session_status="$(curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" \
--max-time "$TASK13_CURL_MAX_TIME" --silent --output "$TASK13_TMP/server-sessions.out" \
--write-out '%{http_code}' \
"http://$frontend/api/sessions")"
[[ "$session_status" == 401 ]] \
|| task13_fail "server session route did not fail closed before OIDC authentication"
if grep -Fq "$TASK13_SECRET_VALUE" "$TASK13_TMP/server-sessions.out"; then
task13_fail "server session failure exposed the fixture secret"
fi
status="$TASK13_TMP/server-auth-status.json"
task13_run_logged "server static OIDC status" "$TASK13_THT" --installation "$TASK13_INSTALLATION" auth status --json
"$TASK13_THT" --installation "$TASK13_INSTALLATION" auth status --json >"$status"
node -e 'const value=JSON.parse(require("fs").readFileSync(process.argv[1], "utf8")); if(value.mode!=="oidc"||!/^sha256:[0-9a-f]{64}$/.test(value.configRevision)) process.exit(1)' "$status" \
|| task13_fail "server static OIDC status was not valid"
diagnostics="$TASK13_TMP/server-auth-diagnostics.json"
task13_run_logged "server live OIDC diagnostics" "$TASK13_THT" --installation "$TASK13_INSTALLATION" auth check --json
"$TASK13_THT" --installation "$TASK13_INSTALLATION" auth check --json >"$diagnostics"
node -e 'const value=JSON.parse(require("fs").readFileSync(process.argv[1], "utf8")); if(value.mode!=="oidc"||value.ready!==true||value.checks.length!==1||value.checks[0].code!=="auth_ready") process.exit(1)' "$diagnostics" \
|| task13_fail "scoped fake OIDC provider did not pass live production diagnostics"
provider_requests="$(docker logs "$TASK13_OIDC_CONTAINER" 2>>"$TASK13_LOG")"
[[ "$(grep -Fc 'group:task13-users' <<<"$provider_requests")" -ge 1 ]] \
|| task13_fail "live OIDC diagnostics did not verify the mandatory user group"
[[ "$(grep -Fc 'group:task13-admins' <<<"$provider_requests")" -ge 1 ]] \
|| task13_fail "live OIDC diagnostics did not verify the mandatory administrator group"
if grep -Fq 'group:task13-unrelated' <<<"$provider_requests" \
|| grep -Fq 'task13-unrelated' "$diagnostics"; then
task13_fail "live OIDC diagnostics queried or warned about an unrelated group"
fi
if grep -Fq "$TASK13_OIDC_CLIENT_SECRET" "$diagnostics" || grep -Fq "$TASK13_AUTHENTIK_API_TOKEN" "$diagnostics"; then
task13_fail "OIDC diagnostics exposed a fixture secret"
fi
}
task13_registry_status() {
task13_authenticated_get workspace-registry/status
}
task13_registry_head() {
sed -n 's/.*"head":"\([0-9a-f][0-9a-f]*\)".*/\1/p'
}
task13_active_registry_head() {
task13_compose exec -T core sed -n \
's/.*"head":"\([0-9a-f][0-9a-f]*\)".*/\1/p' \
/data/workspace-registry/state/active.json
}
task13_assert_sentinels() {
task13_compose exec -T core sh -ceu '
test "$(cat /data/settings/task13-settings)" = settings-preserved
test "$(cat /data/sessions/task13-session)" = sessions-preserved
test "$(cat /home/thoth/.pi/task13-pi-state)" = pi-state-preserved
test -f /data/workspace-registry/state/active.json
'
}
task13_mount_fingerprint() {
docker inspect --format '{{range .Mounts}}{{println .Destination "=" .Type ":" .Name}}{{end}}' "$(task13_core_id)" \
| LC_ALL=C sort
}
task13_service_mount_fingerprint() {
local service="$1"
docker inspect --format '{{range .Mounts}}{{println .Destination "=" .Type ":" .Name}}{{end}}' \
"$(task13_compose ps -q "$service")" | LC_ALL=C sort
}
task13_prepare_persistence() {
task13_compose exec -T core sh -ceu '
printf %s settings-preserved > /data/settings/task13-settings
printf %s sessions-preserved > /data/sessions/task13-session
printf %s pi-state-preserved > /home/thoth/.pi/task13-pi-state
'
TASK13_INITIAL_MOUNTS="$(task13_mount_fingerprint)"
TASK13_INITIAL_HEAD="$(task13_active_registry_head)"
[[ "$TASK13_INITIAL_HEAD" =~ ^[0-9a-f]{40}$ ]] || task13_fail "initial registry head is invalid"
task13_authenticated_get workspaces \
| grep -Fq 'Task 13 Smoke' || task13_fail "initial workspace is unavailable"
}
task13_registry_lifecycle() {
local offline_status offline_head valid_head evidence_head repaired_head
printf '== Recreate offline and retain the validated registry snapshot ==
'
task13_write_environment /fixtures/offline.git
task13_compose_logged "offline Compose recreation" up --detach --force-recreate --wait --wait-timeout 120
offline_status="$(task13_registry_status)"
offline_head="$(printf '%s' "$offline_status" | task13_registry_head)"
[[ "$offline_head" == "$TASK13_INITIAL_HEAD" ]] || task13_fail "offline recreation changed registry head"
grep -Fq '"degraded":true' <<<"$offline_status" || task13_fail "offline recreation did not report degraded mode"
task13_assert_sentinels
[[ "$(task13_mount_fingerprint)" == "$TASK13_INITIAL_MOUNTS" ]] || task13_fail "offline recreation changed volume identity"
printf '== Pull a valid catalog+descriptor metadata update ==
'
task13_replace_once "$TASK13_SEED/$TASK13_WORKSPACE_ID/workspace.yaml" 'name: Task 13 Smoke' 'name: Task 13 Smoke Updated'
task13_replace_once "$TASK13_SEED/thoth-workspaces.yaml" 'name: Task 13 Smoke' 'name: Task 13 Smoke Updated'
task13_commit_registry_change 'Update Task 13 workspace metadata'
task13_write_environment /fixtures/remote.git
task13_compose_logged "online Compose recreation" up --detach --force-recreate --wait --wait-timeout 120
task13_authenticated_post workspace-registry/pull >/dev/null
task13_authenticated_get workspaces | grep -Fq 'Task 13 Smoke Updated' || task13_fail "valid Git update was not activated"
valid_head="$(task13_active_registry_head)"
[[ "$valid_head" =~ ^[0-9a-f]{40}$ && "$valid_head" != "$TASK13_INITIAL_HEAD" ]] || task13_fail "valid Git update did not advance the registry head"
TASK13_INITIAL_HEAD="$valid_head"
task13_assert_sentinels
printf '== Pull a content-only Git Evidence update ==
'
printf 'guide v2\n' >"$TASK13_SEED/$TASK13_WORKSPACE_ID/evidence/guide.md"
task13_commit_registry_change 'Update Task 13 workspace evidence only'
task13_authenticated_post workspace-registry/pull >/dev/null
evidence_head="$(task13_active_registry_head)"
[[ "$evidence_head" =~ ^[0-9a-f]{40}$ && "$evidence_head" != "$valid_head" ]] || task13_fail "content-only Git Evidence update did not advance the registry head"
TASK13_INITIAL_HEAD="$evidence_head"
task13_assert_sentinels
printf '== Reject catalog/descriptor metadata mismatch and retain the valid snapshot ==
'
task13_replace_once "$TASK13_SEED/thoth-workspaces.yaml" 'name: Task 13 Smoke Updated' 'name: Task 13 Smoke Drift'
task13_commit_registry_change 'Break Task 13 workspace metadata parity'
if task13_authenticated_post workspace-registry/pull >>"$TASK13_LOG" 2>&1; then
task13_fail "registry accepted catalog/descriptor metadata mismatch"
fi
[[ "$(task13_active_registry_head)" == "$TASK13_INITIAL_HEAD" ]] || task13_fail "metadata mismatch replaced the valid registry head"
task13_authenticated_get workspaces | grep -Fq 'Task 13 Smoke Updated' || task13_fail "metadata mismatch displaced the valid workspace"
task13_replace_once "$TASK13_SEED/thoth-workspaces.yaml" 'name: Task 13 Smoke Drift' 'name: Task 13 Smoke Updated'
task13_commit_registry_change 'Restore Task 13 workspace metadata parity'
task13_authenticated_post workspace-registry/pull >/dev/null
repaired_head="$(task13_active_registry_head)"
[[ "$repaired_head" =~ ^[0-9a-f]{40}$ && "$repaired_head" != "$TASK13_INITIAL_HEAD" ]] || task13_fail "metadata repair did not restore a fresh valid registry head"
TASK13_INITIAL_HEAD="$repaired_head"
printf '== Reject orphan descriptor directories not listed in the catalog ==
'
mkdir -p "$TASK13_SEED/orphan"
cp "$TASK13_SEED/$TASK13_WORKSPACE_ID/workspace.yaml" "$TASK13_SEED/orphan/workspace.yaml"
task13_replace_once "$TASK13_SEED/orphan/workspace.yaml" 'id: task13-smoke' 'id: orphan'
task13_replace_once "$TASK13_SEED/orphan/workspace.yaml" 'name: Task 13 Smoke Updated' 'name: Orphan Workspace'
task13_replace_once "$TASK13_SEED/orphan/workspace.yaml" 'collection: task13-smoke' 'collection: orphan'
task13_replace_once "$TASK13_SEED/orphan/workspace.yaml" 'uri: task13-smoke/evidence' 'uri: orphan/evidence'
mkdir -p "$TASK13_SEED/orphan/evidence"
printf 'orphan guide\n' >"$TASK13_SEED/orphan/evidence/guide.md"
task13_commit_registry_change 'Add orphan Task 13 workspace directory'
if task13_authenticated_post workspace-registry/pull >>"$TASK13_LOG" 2>&1; then
task13_fail "registry accepted orphan Task 13 descriptor directory"
fi
[[ "$(task13_active_registry_head)" == "$TASK13_INITIAL_HEAD" ]] || task13_fail "orphan descriptor directory replaced the valid registry head"
task13_authenticated_get workspaces | grep -Fq 'Task 13 Smoke Updated' || task13_fail "orphan descriptor displaced the valid workspace"
rm -rf "$TASK13_SEED/orphan"
task13_commit_registry_change 'Remove orphan Task 13 workspace directory'
task13_authenticated_post workspace-registry/pull >/dev/null
TASK13_INITIAL_HEAD="$(task13_active_registry_head)"
printf '== Reject the retired flat workspace layout and retain the valid snapshot ==
'
mkdir -p "$TASK13_SEED/workspaces" "$TASK13_SEED/workspace-content/$TASK13_WORKSPACE_ID/evidence"
cp "$TASK13_ROOT/scripts/fixtures/workspace-registry-task13.yaml" "$TASK13_SEED/workspaces/$TASK13_WORKSPACE_ID.yaml"
printf 'legacy guide\n' >"$TASK13_SEED/workspace-content/$TASK13_WORKSPACE_ID/evidence/guide.md"
task13_commit_registry_change 'Reintroduce retired flat Task 13 workspace layout'
if task13_authenticated_post workspace-registry/pull >>"$TASK13_LOG" 2>&1; then
task13_fail "registry accepted the retired flat Task 13 workspace layout"
fi
[[ "$(task13_active_registry_head)" == "$TASK13_INITIAL_HEAD" ]] || task13_fail "retired flat workspace layout replaced the valid registry head"
task13_authenticated_get workspaces | grep -Fq 'Task 13 Smoke Updated' || task13_fail "retired flat workspace layout displaced the valid workspace"
task13_assert_sentinels
}
task13_prepare_bad_candidate() {
task13_run_logged "pull pinned stopped-core candidate" docker image pull "$TASK13_BAD_CANDIDATE_IMAGE"
TASK13_BAD_CANDIDATE_ID="$(docker image inspect --format '{{.Id}}' "$TASK13_BAD_CANDIDATE_IMAGE")"
[[ "$TASK13_BAD_CANDIDATE_ID" =~ ^sha256:[0-9a-f]{64}$ ]] \
|| task13_fail "bad candidate image identity was not resolved"
task13_run_logged "prove bad candidate exits" docker run \
--name "$TASK13_BAD_CANDIDATE_CONTAINER" \
--label "io.thothii.task13.run=$TASK13_RUN_ID" \
"$TASK13_BAD_CANDIDATE_IMAGE"
[[ "$(docker container inspect --format '{{.State.Running}}:{{.State.ExitCode}}' \
"$TASK13_BAD_CANDIDATE_CONTAINER")" == false:0 ]] \
|| task13_fail "bad candidate did not reach the guaranteed stopped state"
task13_remove_labeled_container "$TASK13_BAD_CANDIDATE_CONTAINER"
}
task13_update_rollback() {
local before_image before_mounts before_head output rc phase after_image after_mounts after_head
printf '== Inject a bad pinned Pi candidate and prove automatic rollback ==\n'
task13_prepare_bad_candidate
before_image="$(docker inspect --format '{{.Image}}' "$(task13_core_id)")"
TASK13_PREVIOUS_IMAGE_ID="$before_image"
before_mounts="$(task13_mount_fingerprint)"
before_head="$(task13_active_registry_head)"
output="$TASK13_TMP/tht-update.out"
set +e
"$TASK13_THT" --installation "$TASK13_INSTALLATION" pi update \
--version "$TASK13_BAD_PI_VERSION" --source pull --image "$TASK13_BAD_CANDIDATE_IMAGE" --yes \
>"$output" 2>&1
rc=$?
set -e
[[ "$rc" -ne 0 ]] || task13_fail "bad Pi candidate unexpectedly passed update verification"
if grep -Fq "$TASK13_SECRET_VALUE" "$output"; then
task13_fail "tht update output exposed the fixture secret"
fi
grep -Fq 'previous core image was restored' "$output" \
|| { task13_sanitize <"$output" >&2; task13_fail "tht did not report automatic rollback"; }
[[ -f "$TASK13_UPDATE_STATE" ]] || task13_fail "tht update state was not persisted"
phase="$(sed -n 's/.*"phase": "\([^"]*\)".*/\1/p' "$TASK13_UPDATE_STATE" | head -n 1)"
[[ "$phase" == rolled_back ]] || task13_fail "update state phase is not rolled_back"
if grep -Fq "$TASK13_SECRET_VALUE" "$TASK13_UPDATE_STATE"; then
task13_fail "update state exposed the fixture secret"
fi
task13_compose_files
after_image="$(docker inspect --format '{{.Image}}' "$(task13_core_id)")"
after_mounts="$(task13_mount_fingerprint)"
after_head="$(task13_active_registry_head)"
[[ "$after_image" == "$before_image" ]] || task13_fail "rollback did not restore the previous core image"
[[ "$after_mounts" == "$before_mounts" ]] || task13_fail "rollback changed persistence volume identity"
[[ "$after_head" == "$before_head" ]] || task13_fail "rollback changed the active registry revision"
task13_assert_sentinels
task13_run_logged "post-rollback tht doctor" \
"$TASK13_THT" --installation "$TASK13_INSTALLATION" pi doctor
task13_authenticated_get workspaces \
| grep -Fq 'Task 13 Smoke' || task13_fail "rollback lost the active workspace"
}
task13_remove_labeled_container() {
local name="$1" label
[[ -n "$name" ]] || return 0
if ! docker container inspect "$name" >/dev/null 2>&1; then
return 0
fi
label="$(docker container inspect --format '{{ index .Config.Labels "io.thothii.task13.run" }}' "$name")"
[[ "$label" == "$TASK13_RUN_ID" ]] || {
printf 'refusing to remove foreign container %s\n' "$name" >&2
return 1
}
task13_bounded "$TASK13_CLEANUP_TIMEOUT" "remove owned container" \
docker container rm --force "$name" >/dev/null
}
task13_remove_labeled_image() {
local reference="$1" label
[[ -n "$reference" ]] || return 0
if ! docker image inspect "$reference" >/dev/null 2>&1; then
return 0
fi
label="$(docker image inspect --format '{{ index .Config.Labels "io.thothii.task13.run" }}' "$reference")"
[[ "$label" == "$TASK13_RUN_ID" ]] || {
printf 'refusing to remove foreign image %s\n' "$reference" >&2
return 1
}
task13_bounded "$TASK13_CLEANUP_TIMEOUT" "remove owned image" \
docker image rm "$reference" >/dev/null
}
task13_remove_transaction_image() {
local reference="$1" expected_id="$2" actual_id
[[ -n "$reference" ]] || return 0
if ! docker image inspect "$reference" >/dev/null 2>&1; then
return 0
fi
if [[ ! "$reference" =~ ^thothii-core:tht-[0-9a-f]{16}-(candidate|previous)$ \
|| ! "$expected_id" =~ ^sha256:([0-9a-f]{64}|owned)$ ]]; then
printf 'refusing to remove invalid transaction image reference %s\n' "$reference" >&2
return 1
fi
actual_id="$(docker image inspect --format '{{.Id}}' "$reference")"
[[ "$actual_id" == "$expected_id" ]] || {
printf 'refusing to remove foreign transaction image %s\n' "$reference" >&2
return 1
}
task13_bounded "$TASK13_CLEANUP_TIMEOUT" "remove owned transaction image" \
docker image rm "$reference" >/dev/null
}
task13_assert_project_ownership() {
local kind id ids label
for kind in container volume network; do
if [[ "$kind" == container ]]; then
if ! ids="$(docker container ls -aq --filter "label=com.docker.compose.project=$TASK13_PROJECT")"; then
task13_fail "could not enumerate Compose project container resources"
return 1
fi
elif ! ids="$(docker "$kind" ls -q --filter "label=com.docker.compose.project=$TASK13_PROJECT")"; then
task13_fail "could not enumerate Compose project $kind resources"
return 1
fi
while IFS= read -r id; do
[[ -n "$id" ]] || continue
case "$kind" in
container)
if ! label="$(docker container inspect --format '{{ index .Config.Labels "io.thothii.task13.run" }}' "$id")"; then
task13_fail "could not inspect Compose project container resource"
return 1
fi
;;
volume)
if ! label="$(docker volume inspect --format '{{ index .Labels "io.thothii.task13.run" }}' "$id")"; then
task13_fail "could not inspect Compose project volume resource"
return 1
fi
;;
network)
if ! label="$(docker network inspect --format '{{ index .Labels "io.thothii.task13.run" }}' "$id")"; then
task13_fail "could not inspect Compose project network resource"
return 1
fi
;;
esac
if [[ "$label" != "$TASK13_RUN_ID" ]]; then
task13_fail "Compose project contains a foreign $kind resource"
return 1
fi
done <<<"$ids"
done
}
task13_assert_built_image_ownership() {
local image label
for image in "$TASK13_CORE_IMAGE" "$TASK13_FRONTEND_IMAGE"; do
label="$(docker image inspect --format '{{ index .Config.Labels "io.thothii.task13.run" }}' "$image")"
[[ "$label" == "$TASK13_RUN_ID" ]] || task13_fail "built image lacks the Task 13 run label"
done
}
task13_cleanup() {
local original_rc="$1" cleanup_rc=0 transaction="" leftovers="" image_id=""
set +e
if [[ "$original_rc" -ne 0 && "${TASK13_FAILURE_LOGGED:-0}" -eq 0 ]] \
&& [[ -n "${TASK13_LOG:-}" && -f "$TASK13_LOG" ]]; then
printf '%s\n' '--- sanitized Task 13 diagnostic log ---' >&2
tail -n 200 "$TASK13_LOG" | task13_sanitize >&2
fi
task13_remove_labeled_container "${TASK13_BAD_CANDIDATE_CONTAINER:-}" || cleanup_rc=1
task13_remove_labeled_container "${TASK13_LLM_CONTAINER:-}" || cleanup_rc=1
task13_remove_labeled_container "${TASK13_OIDC_CONTAINER:-}" || cleanup_rc=1
if [[ -n "${TASK13_PROJECT:-}" && -n "${TASK13_ROOT:-}" && -f "${TASK13_OVERRIDE:-}" ]]; then
if task13_assert_project_ownership >>"${TASK13_LOG:-/dev/null}" 2>&1; then
task13_compose_files
task13_bounded "$TASK13_CLEANUP_TIMEOUT" "stop owned Compose project" \
"${TASK13_COMPOSE[@]}" down --volumes --remove-orphans --timeout 10 \
>>"${TASK13_LOG:-/dev/null}" 2>&1 || cleanup_rc=1
else
cleanup_rc=1
fi
fi
if [[ -f "${TASK13_UPDATE_STATE:-}" ]]; then
transaction="$(sed -n 's/.*"transaction": "\([^"]*\)".*/\1/p' "$TASK13_UPDATE_STATE" | head -n 1)"
fi
if [[ -n "$transaction" ]]; then
task13_remove_transaction_image "thothii-core:tht-$transaction-candidate" \
"${TASK13_BAD_CANDIDATE_ID:-}" >>"${TASK13_LOG:-/dev/null}" 2>&1 || cleanup_rc=1
task13_remove_transaction_image "thothii-core:tht-$transaction-previous" \
"${TASK13_PREVIOUS_IMAGE_ID:-}" >>"${TASK13_LOG:-/dev/null}" 2>&1 || cleanup_rc=1
fi
for image in \
"${TASK13_FRONTEND_IMAGE:-}" \
"${TASK13_CORE_IMAGE:-}"; do
[[ -n "$image" ]] || continue
task13_remove_labeled_image "$image" >>"${TASK13_LOG:-/dev/null}" 2>&1 || cleanup_rc=1
done
if [[ -n "${TASK13_RUN_ID:-}" ]]; then
while IFS= read -r image_id; do
[[ -n "$image_id" ]] || continue
task13_remove_labeled_image "$image_id" >>"${TASK13_LOG:-/dev/null}" 2>&1 || cleanup_rc=1
done < <(docker image ls -q --filter "label=io.thothii.task13.run=$TASK13_RUN_ID" | sort -u)
fi
if [[ -n "${TASK13_CONTROL_DIR:-}" ]]; then
if [[ "$TASK13_CONTROL_DIR" == "$TASK13_ROOT/.tht/$TASK13_PROJECT" \
&& "$TASK13_PROJECT" =~ ^thothii-[0-9a-f]{12}$ ]]; then
rm -rf "$TASK13_CONTROL_DIR"
else
cleanup_rc=1
fi
fi
if [[ -n "${TASK13_RUN_ID:-}" ]]; then
leftovers="$(docker container ls -aq --filter "label=io.thothii.task13.run=$TASK13_RUN_ID")"
leftovers+="$(docker volume ls -q --filter "label=io.thothii.task13.run=$TASK13_RUN_ID")"
leftovers+="$(docker network ls -q --filter "label=io.thothii.task13.run=$TASK13_RUN_ID")"
leftovers+="$(docker image ls -q --filter "label=io.thothii.task13.run=$TASK13_RUN_ID")"
[[ -z "$leftovers" ]] || cleanup_rc=1
fi
if [[ -n "${TASK13_TMP:-}" && -d "$TASK13_TMP" ]]; then
if [[ "${TASK13_TMP%/*}" == "${TASK13_TMP_PARENT:-}" \
&& "${TASK13_TMP##*/}" == thothii-task13.* ]]; then
rm -rf "$TASK13_TMP"
else
cleanup_rc=1
fi
fi
if [[ "$cleanup_rc" -eq 0 ]]; then
printf 'Task 13 cleanup proof: no labeled containers, volumes, networks, or images remain for %s.\n' \
"${TASK13_RUN_ID:-unknown}"
else
printf 'Task 13 cleanup proof failed for %s.\n' "${TASK13_RUN_ID:-unknown}" >&2
fi
trap - EXIT
if [[ "$original_rc" -ne 0 ]]; then
exit "$original_rc"
fi
exit "$cleanup_rc"
}
task13_self_test_sanitizer() {
local input output leaked
TASK13_SECRET_VALUE="fixture-known-secret"
input="$(printf '%s\n' \
'fixture-known-secret' \
'password=plain-secret' \
'{"api_key":"json-secret"}' \
'Authorization: Bearer bearer-secret' \
'https://alice:url-secret@example.invalid/repo.git')"
output="$(printf '%s\n' "$input" | task13_sanitize)"
for leaked in fixture-known-secret plain-secret json-secret bearer-secret url-secret; do
if grep -Fq "$leaked" <<<"$output"; then
task13_fail "sanitizer leaked $leaked"
fi
done
[[ "$(grep -Fc '[REDACTED]' <<<"$output")" -eq 5 ]] \
|| task13_fail "sanitizer did not redact every credential form"
}
task13_self_test_server_workspace_diagnostics() {
local response output count i=1
response="$(mktemp "${TMPDIR:-/tmp}/thothii-task13-server-response.XXXXXX")"
TASK13_SECRET_VALUE="fixture-known-secret"
printf '%s\n' \
'{"error":"workspace_invalid","detail":"password=fixture-known-secret"}' >"$response"
task13_compose() {
if [[ "$*" == "exec -T core node --input-type=module -e "* ]]; then
printf '%s\n' \
'{"name":"WorkspaceRegistryError","code":"workspace_invalid","message":"Workspace snapshot integrity check failed"}'
return 0
fi
[[ "$*" == "logs --no-color --tail 100 core" ]] \
|| task13_fail "server diagnostics requested an unexpected Compose command"
while [[ "$i" -le 150 ]]; do
printf 'core-log-%03d token=fixture-known-secret\n' "$i"
i=$((i + 1))
done
}
if ! output="$(task13_report_server_workspace_failure 400 "$response" 2>&1)"; then
unset -f task13_compose
rm -f "$response"
task13_fail "server workspace diagnostics could not be captured"
fi
unset -f task13_compose
rm -f "$response"
[[ "$output" == *'authenticated server /api/workspaces returned HTTP 400'* ]] \
|| task13_fail "server diagnostics omit the unexpected HTTP status"
[[ "$output" == *'workspace_invalid'* ]] \
|| task13_fail "server diagnostics omit the generic response"
[[ "$output" == *'Workspace snapshot integrity check failed'* ]] \
|| task13_fail "server diagnostics omit the bounded internal registry reason"
[[ "$output" == *'core-log-051'* && "$output" != *'core-log-050'* ]] \
|| task13_fail "server diagnostics do not bound core logs to the last 100 lines"
count="$(grep -Ec '^core-log-[0-9]{3}' <<<"$output")"
[[ "$count" -eq 100 ]] || task13_fail "server diagnostics emitted $count core log lines"
[[ "$output" != *'fixture-known-secret'* ]] \
|| task13_fail "server diagnostics leaked the fixture secret"
[[ "$(grep -Fc '[REDACTED]' <<<"$output")" -ge 101 ]] \
|| task13_fail "server diagnostics did not sanitize response and core logs"
}
task13_self_test_core_startup_diagnostics() {
local output
TASK13_SECRET_VALUE="fixture-known-secret"
task13_compose() {
case "$*" in
"ps --all -q core") printf '%s\n' 'task13-core-id' ;;
"logs --no-color --tail 100 core")
printf '%s\n' \
'Error: EACCES: permission denied, mkdir /data/auth/sessions' \
'password=plain-secret token=fixture-known-secret' \
'secret path: /run/secrets/private-token' \
' at createFileAuthSessionStore (/app/backend/dist/auth/session-store.js:101:9)'
;;
*) task13_fail "startup diagnostics requested an unexpected Compose command: $*" ;;
esac
}
docker() {
[[ "$*" == "inspect --format {{.State.Status}}:{{.State.ExitCode}} task13-core-id" ]] \
|| task13_fail "startup diagnostics requested an unexpected Docker command: $*"
printf '%s\n' 'exited:1'
}
output="$(task13_report_core_startup_failure 2>&1)"
unset -f task13_compose docker
[[ "$output" == 'Core startup cause: authentication state storage is unavailable (exit code 1).' ]] \
|| task13_fail "startup diagnostics emitted a non-allowlisted cause: $output"
for leaked in EACCES permission /data/auth /run/secrets plain-secret fixture-known-secret \
createFileAuthSessionStore session-store.js; do
[[ "$output" != *"$leaked"* ]] || task13_fail "startup diagnostics leaked $leaked"
done
}
task13_self_test_cleanup_ownership() {
local calls foreign_error owned_name foreign_name
calls="$(mktemp "${TMPDIR:-/tmp}/thothii-task13-cleanup-contract.XXXXXX")"
foreign_error="$calls.foreign-error"
owned_name="task13-owned-contract"
foreign_name="task13-foreign-contract"
TASK13_RUN_ID="task13-contract-run"
docker() {
printf '%s\n' "$*" >>"$calls"
if [[ "$1 $2" == "container inspect" ]]; then
if [[ "$3" == "--format" ]]; then
if [[ "${*: -1}" == "$owned_name" ]]; then
printf '%s\n' "$TASK13_RUN_ID"
else
printf '%s\n' 'some-other-run'
fi
fi
return 0
fi
[[ "$1 $2" == "container rm" ]]
}
if task13_remove_labeled_container "$foreign_name" 2>"$foreign_error"; then
unset -f docker
rm -f "$calls" "$foreign_error"
task13_fail "cleanup accepted a foreign-labeled container"
fi
if grep -Fq "container rm --force $foreign_name" "$calls"; then
unset -f docker
rm -f "$calls" "$foreign_error"
task13_fail "cleanup attempted to remove a foreign-labeled container"
fi
grep -Fq "refusing to remove foreign container $foreign_name" "$foreign_error" \
|| task13_fail "cleanup refusal was not explicit"
task13_remove_labeled_container "$owned_name"
[[ "$(grep -Fc "container rm --force $owned_name" "$calls")" -eq 1 ]] \
|| task13_fail "cleanup did not remove exactly the owned container"
unset -f docker
rm -f "$calls" "$foreign_error"
}
task13_self_test_image_cleanup_ownership() {
local calls foreign_error owned_ref foreign_ref
calls="$(mktemp "${TMPDIR:-/tmp}/thothii-task13-image-cleanup-contract.XXXXXX")"
foreign_error="$calls.foreign-error"
owned_ref="task13-owned-contract:local"
foreign_ref="task13-foreign-contract:local"
TASK13_RUN_ID="task13-contract-run"
if ! declare -F task13_remove_labeled_image >/dev/null; then
rm -f "$calls" "$foreign_error"
task13_fail "image cleanup ownership guard is missing"
fi
docker() {
printf '%s\n' "$*" >>"$calls"
if [[ "$1 $2" == "image inspect" ]]; then
if [[ "$3" == "--format" ]]; then
if [[ "${*: -1}" == "$owned_ref" ]]; then
printf '%s\n' "$TASK13_RUN_ID"
else
printf '%s\n' 'some-other-run'
fi
fi
return 0
fi
[[ "$1 $2" == "image rm" ]]
}
if task13_remove_labeled_image "$foreign_ref" 2>"$foreign_error"; then
unset -f docker
rm -f "$calls" "$foreign_error"
task13_fail "cleanup accepted a foreign-labeled image"
fi
if grep -Fq "image rm $foreign_ref" "$calls"; then
unset -f docker
rm -f "$calls" "$foreign_error"
task13_fail "cleanup attempted to remove a foreign-labeled image"
fi
grep -Fq "refusing to remove foreign image $foreign_ref" "$foreign_error" \
|| task13_fail "image cleanup refusal was not explicit"
task13_remove_labeled_image "$owned_ref"
[[ "$(grep -Fc "image rm $owned_ref" "$calls")" -eq 1 ]] \
|| task13_fail "cleanup did not remove exactly the owned image reference"
unset -f docker
rm -f "$calls" "$foreign_error"
}
task13_self_test_transaction_image_cleanup() {
local calls foreign_error owned_ref foreign_ref
calls="$(mktemp "${TMPDIR:-/tmp}/thothii-task13-transaction-cleanup-contract.XXXXXX")"
foreign_error="$calls.foreign-error"
owned_ref="thothii-core:tht-0123456789abcdef-candidate"
foreign_ref="thothii-core:tht-fedcba9876543210-candidate"
if ! declare -F task13_remove_transaction_image >/dev/null; then
rm -f "$calls" "$foreign_error"
task13_fail "transaction image cleanup guard is missing"
fi
docker() {
printf '%s\n' "$*" >>"$calls"
if [[ "$1 $2" == "image inspect" ]]; then
if [[ "$3" == "--format" ]]; then
if [[ "${*: -1}" == "$owned_ref" ]]; then
printf '%s\n' 'sha256:owned'
else
printf '%s\n' 'sha256:foreign'
fi
fi
return 0
fi
[[ "$1 $2" == "image rm" ]]
}
if task13_remove_transaction_image "$foreign_ref" 'sha256:owned' 2>"$foreign_error"; then
unset -f docker
rm -f "$calls" "$foreign_error"
task13_fail "cleanup accepted a transaction image with a foreign identity"
fi
if grep -Fq "image rm $foreign_ref" "$calls"; then
unset -f docker
rm -f "$calls" "$foreign_error"
task13_fail "cleanup attempted to remove a foreign transaction image"
fi
grep -Fq "refusing to remove foreign transaction image $foreign_ref" "$foreign_error" \
|| task13_fail "transaction image cleanup refusal was not explicit"
task13_remove_transaction_image "$owned_ref" 'sha256:owned'
[[ "$(grep -Fc "image rm $owned_ref" "$calls")" -eq 1 ]] \
|| task13_fail "cleanup did not remove exactly the owned transaction image reference"
unset -f docker
rm -f "$calls" "$foreign_error"
}
task13_self_test_rollback_fixture_contract() {
[[ "${TASK13_BAD_CANDIDATE_BEHAVIOR:-}" == stopped ]] \
|| task13_fail "rollback candidate is not declared as guaranteed stopped"
[[ "$TASK13_BAD_CANDIDATE_IMAGE" =~ ^hello-world@sha256:[0-9a-f]{64}$ ]] \
|| task13_fail "rollback candidate is not the digest-pinned stopped fixture"
}
task13_self_test_runtime_binding_fixture() {
local root
root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)"
"$root/scripts/test-task13-runtime-fixtures.sh" local
}
task13_self_test_server_runtime_binding_fixture() {
local root
root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)"
"$root/scripts/test-task13-runtime-fixtures.sh" server
}
task13_self_test_stopped_project_containers() {
local calls foreign_error
calls="$(mktemp "${TMPDIR:-/tmp}/thothii-task13-project-containers.XXXXXX")"
foreign_error="$calls.foreign-error"
TASK13_PROJECT="thothii-0123456789ab"
TASK13_RUN_ID="task13-contract-run"
docker() {
printf '%s\n' "$*" >>"$calls"
case "$1 $2 $3" in
"container ls -aq") printf '%s\n' stopped-foreign ;;
"container inspect --format") printf '%s\n' some-other-run ;;
"volume ls -q"|"network ls -q") : ;;
*) return 1 ;;
esac
}
if task13_assert_project_ownership 2>"$foreign_error"; then
unset -f docker
rm -f "$calls" "$foreign_error"
task13_fail "project cleanup accepted a stopped foreign container"
fi
grep -Fq 'container ls -aq' "$calls" \
|| task13_fail "project cleanup did not enumerate stopped containers"
grep -Fq 'Compose project contains a foreign container resource' "$foreign_error" \
|| task13_fail "stopped foreign container refusal was not explicit"
: >"$calls"
docker() {
printf '%s\n' "$*" >>"$calls"
case "$1 $2 $3" in
"container ls -aq") printf '%s\n' stopped-owned ;;
"container inspect --format") printf '%s\n' "$TASK13_RUN_ID" ;;
"volume ls -q"|"network ls -q") : ;;
*) return 1 ;;
esac
}
task13_assert_project_ownership
[[ "$(grep -Fc 'container inspect --format' "$calls")" -eq 1 ]] \
|| task13_fail "project cleanup did not inspect exactly the stopped owned container"
unset -f docker
rm -f "$calls" "$foreign_error"
}
task13_self_test_timeout_process_group() {
local child_file child_pid="" rc
child_file="$(mktemp "${TMPDIR:-/tmp}/thothii-task13-timeout-child.XXXXXX")"
set +e
task13_bounded 1 "child-process regression" bash -c \
'sleep 30 & printf "%s\n" "$!" >"$1"; wait' _ "$child_file" >/dev/null 2>&1
rc=$?
set -e
child_pid="$(sed -n '1p' "$child_file")"
[[ "$rc" -ne 0 ]] || {
rm -f "$child_file"
task13_fail "timed command unexpectedly succeeded"
}
if [[ -n "$child_pid" ]] && kill -0 "$child_pid" 2>/dev/null; then
kill -KILL "$child_pid" 2>/dev/null || true
rm -f "$child_file"
task13_fail "timed command left its child process alive"
fi
rm -f "$child_file"
}
task13_self_test_nested_timeout_process_group() {
local child_file child_pid="" rc
child_file="$(mktemp "${TMPDIR:-/tmp}/thothii-task13-nested-timeout.XXXXXX")"
task13_nested_timeout_fixture() {
task13_bounded 30 "nested child-process regression" bash -c \
'sleep 30 & printf "%s\n" "$!" >"$1"; wait' _ "$child_file"
}
set +e
task13_supervise 1 "nested timeout supervisor" task13_nested_timeout_fixture >/dev/null 2>&1
rc=$?
set -e
unset -f task13_nested_timeout_fixture
child_pid="$(sed -n '1p' "$child_file")"
[[ "$rc" -ne 0 ]] || {
rm -f "$child_file"
task13_fail "nested timed command unexpectedly succeeded"
}
if [[ -n "$child_pid" ]] && kill -0 "$child_pid" 2>/dev/null; then
kill -KILL "$child_pid" 2>/dev/null || true
rm -f "$child_file"
task13_fail "outer timeout left its nested command child alive"
fi
rm -f "$child_file"
}
task13_self_test_public_timeout_contract() {
local root
root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)"
grep -Eq 'task13_supervise[[:space:]].*task13_smoke_main[[:space:]]+full' \
"$root/scripts/unified-deployment-smoke.sh" \
|| task13_fail "direct unified smoke invocation lacks an internal supervisor"
grep -Eq 'task13_supervise[[:space:]].*task13_smoke_main[[:space:]]+update' \
"$root/scripts/tht-update-smoke.sh" \
|| task13_fail "direct update smoke invocation lacks an internal supervisor"
grep -Eq 'task13_supervise[[:space:]].*task13_internal_semantic_smoke_main' \
"$root/scripts/internal-semantic-smoke.sh" \
|| task13_fail "direct internal semantic smoke invocation lacks an internal supervisor"
}
task13_self_test_internal_semantic_offline_contract() {
local root script
root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)"
script="$root/scripts/internal-semantic-smoke.sh"
grep -Fq 'task13_write_offline_semantic_override' "$script" \
|| task13_fail "internal semantic smoke lacks a dedicated offline override"
grep -Fq 'task13_offline_semantic_compose_logged "offline semantic recreation" \' "$script" \
|| task13_fail "offline semantic recreation must use the isolated offline compose wrapper"
grep -Fq 'up --detach --wait --wait-timeout 120 --pull never qdrant embedding' "$script" \
|| task13_fail "offline semantic recreation must start only qdrant and embedding with --pull never"
grep -Eq 'down --remove-orphans --timeout 10$' "$script" \
|| task13_fail "offline semantic phase must stop the stack before isolated recreation"
grep -Fq 'internal: true' "$script" \
|| task13_fail "offline semantic override must disable network egress"
if grep -Eq 'offline semantic recreation.*embedding-model-init|offline semantic recreation.*core|offline semantic recreation.*frontend' "$script"; then
task13_fail "offline semantic recreation must exclude bootstrap and non-semantic services"
fi
}
task13_self_test_windows_release_contract() {
local root script workflow
root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)"
script="$root/scripts/test-windows-clone-contract.ps1"
workflow="$root/.github/workflows/deployment.yml"
grep -Fq 'Task 13 path with spaces' "$script" \
|| task13_fail "Windows contract does not operate from a path containing spaces"
grep -Fq 'DockerStartup' "$script" \
|| task13_fail "Windows contract lacks an explicit Docker startup mode"
grep -Fq 'Kill($true)' "$script" \
|| task13_fail "Windows bounded runner does not kill the full process tree"
grep -Fq 'docker-desktop' "$workflow" \
|| task13_fail "workflow lacks a manual self-hosted Windows Docker Desktop gate"
grep -Fq -- '-DockerStartup' "$workflow" \
|| task13_fail "manual Windows release job does not execute Docker startup mode"
}
task13_self_test_server_release_contract() {
local root workflow server_smoke
root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)"
workflow="$root/.github/workflows/deployment.yml"
server_smoke="$root/scripts/server-deployment-smoke.sh"
[[ -x "$server_smoke" ]] || task13_fail "bounded Linux server deployment smoke is missing"
grep -Fq 'deploy/compose.server.yaml' "$root/scripts/unified-deployment-smoke.sh" \
|| task13_fail "server smoke does not load the server profile"
grep -Fq 'deploy/compose.session-server.yaml.example' "$root/scripts/unified-deployment-smoke.sh" \
|| task13_fail "server smoke does not load the required session overlay"
grep -Eq 'timeout .*scripts/server-deployment-smoke\.sh' "$workflow" \
|| task13_fail "workflow lacks an outer timeout for the Linux server smoke"
}
task13_self_test_source_contract() {
local root host_network push_command registry_function workflow uses_count pinned_uses_count
root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)"
workflow="$root/.github/workflows/deployment.yml"
host_network='--network'' host'
push_command='docker image ''push'
registry_function='task13_start_''registry'
if rg -n 'docker[[:space:]]+(system[[:space:]]+)?prune' \
"$root/scripts/unified-deployment-smoke.sh" \
"$root/scripts/tht-update-smoke.sh" \
"$root/scripts/internal-semantic-smoke.sh" >/dev/null; then
task13_fail "Task 13 smoke scripts must never prune global Docker state"
fi
! grep -Fq -- "$host_network" "$root/scripts/unified-deployment-smoke.sh" \
|| task13_fail "the image registry must not depend on host networking"
if grep -Fq -- "$push_command" "$root/scripts/unified-deployment-smoke.sh" \
|| grep -Fq -- "$registry_function" "$root/scripts/unified-deployment-smoke.sh"; then
task13_fail "the rollback fixture must not depend on a daemon-to-host local image registry"
fi
grep -Eq '^TASK13_BAD_CANDIDATE_IMAGE="[^"[:space:]]+@sha256:[0-9a-f]{64}"$' \
"$root/scripts/unified-deployment-smoke.sh" \
|| task13_fail "the bad rollback candidate must be an immutable digest reference"
grep -Eq 'timeout .*scripts/unified-deployment-smoke\.sh' "$workflow" \
|| task13_fail "CI lacks an outer timeout for the unified deployment smoke"
grep -Eq 'timeout .*scripts/tht-update-smoke\.sh' "$workflow" \
|| task13_fail "CI lacks an outer timeout for the tht update smoke"
uses_count="$(grep -Ec '^[[:space:]]+uses:' "$workflow")"
pinned_uses_count="$(grep -Ec '^[[:space:]]+uses: [^[:space:]]+@[0-9a-f]{40}([[:space:]]|$)' "$workflow")"
[[ "$uses_count" -gt 0 && "$uses_count" -eq "$pinned_uses_count" ]] \
|| task13_fail "every deployment workflow action must use a full immutable commit pin"
if grep -Fq '${{ secrets.' "$workflow"; then
task13_fail "the deployment release gate must not require repository secrets"
fi
for command in \
'bash scripts/verify-line-endings.sh' \
'bash scripts/test-unified-compose.sh' \
'bash scripts/test-compose-secret-policy.sh' \
'bash scripts/test-no-deployment-coupling.sh' \
'bash scripts/test-verify-workspace-install-docs.sh' \
'npx vitest run' \
'npx tsc --noEmit -p .' \
'npx tsc -b' \
'./scripts/test-windows-clone-contract.ps1'; do
grep -Fq "$command" "$workflow" || task13_fail "CI coverage is missing: $command"
done
}
task13_self_test() {
task13_self_test_sanitizer
task13_self_test_core_startup_diagnostics
task13_self_test_server_workspace_diagnostics
task13_self_test_cleanup_ownership
task13_self_test_image_cleanup_ownership
task13_self_test_transaction_image_cleanup
task13_self_test_rollback_fixture_contract
task13_self_test_runtime_binding_fixture
task13_self_test_server_runtime_binding_fixture
task13_self_test_stopped_project_containers
task13_self_test_timeout_process_group
task13_self_test_nested_timeout_process_group
task13_self_test_public_timeout_contract
task13_self_test_internal_semantic_offline_contract
task13_self_test_windows_release_contract
task13_self_test_server_release_contract
task13_self_test_source_contract
printf 'Task 13 smoke safety contracts passed.\n'
}
task13_self_test_case() {
case "$1" in
rollback) task13_self_test_rollback_fixture_contract ;;
runtime-bindings) task13_self_test_runtime_binding_fixture ;;
server-bindings) task13_self_test_server_runtime_binding_fixture ;;
cleanup) task13_self_test_stopped_project_containers ;;
timeout-group) task13_self_test_timeout_process_group ;;
timeout-nested) task13_self_test_nested_timeout_process_group ;;
timeout-public) task13_self_test_public_timeout_contract ;;
semantic-offline) task13_self_test_internal_semantic_offline_contract ;;
windows) task13_self_test_windows_release_contract ;;
server) task13_self_test_server_release_contract ;;
server-diagnostics) task13_self_test_server_workspace_diagnostics ;;
startup-diagnostics) task13_self_test_core_startup_diagnostics ;;
*) task13_fail "unknown Task 13 self-test case: $1" ;;
esac
}
task13_fixtures_only() {
local tmp descriptor_path catalog_path initial_head updated_head
tmp="$(mktemp -d "${TMPDIR:-/tmp}/thothii-task13-fixtures.XXXXXX")"
trap 'rm -rf "$tmp"' RETURN
TASK13_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)"
TASK13_TMP="$tmp"
TASK13_REMOTE="$tmp/remote.git"
TASK13_SEED="$tmp/seed"
TASK13_BRANCH="task13-smoke"
TASK13_LOG="$tmp/task13.log"
TASK13_FAILURE_LOGGED=0
: >"$TASK13_LOG"
task13_seed_registry
descriptor_path="$TASK13_SEED/$TASK13_WORKSPACE_ID/workspace.yaml"
catalog_path="$TASK13_SEED/thoth-workspaces.yaml"
[[ -f "$catalog_path" ]] || task13_fail "missing Task 13 root workspace catalog"
[[ -f "$descriptor_path" ]] || task13_fail "missing Task 13 nested workspace descriptor"
[[ -f "$TASK13_SEED/$TASK13_WORKSPACE_ID/evidence/guide.md" ]] || task13_fail "missing Task 13 nested workspace evidence"
[[ ! -e "$TASK13_SEED/workspaces/$TASK13_WORKSPACE_ID.yaml" ]] || task13_fail "legacy Task 13 flat descriptor path exists"
[[ ! -e "$TASK13_SEED/workspace-content/$TASK13_WORKSPACE_ID" ]] || task13_fail "legacy Task 13 flat evidence path exists"
initial_head="$(git -C "$TASK13_SEED" rev-parse HEAD)"
task13_replace_once "$descriptor_path" 'name: Task 13 Smoke' 'name: Task 13 Smoke Updated'
task13_replace_once "$catalog_path" 'name: Task 13 Smoke' 'name: Task 13 Smoke Updated'
task13_commit_registry_change 'Update Task 13 workspace metadata'
updated_head="$(git -C "$TASK13_SEED" rev-parse HEAD)"
[[ "$updated_head" != "$initial_head" ]] || task13_fail "Task 13 metadata update did not advance Git head"
printf 'guide v2\n' >"$TASK13_SEED/$TASK13_WORKSPACE_ID/evidence/guide.md"
task13_commit_registry_change 'Update Task 13 workspace evidence only'
[[ "$(git -C "$TASK13_SEED" rev-parse HEAD)" != "$updated_head" ]] || task13_fail "Task 13 evidence-only commit did not advance Git head"
mkdir -p "$TASK13_SEED/workspaces" "$TASK13_SEED/workspace-content/$TASK13_WORKSPACE_ID/evidence"
cp "$TASK13_ROOT/scripts/fixtures/workspace-registry-task13.yaml" "$TASK13_SEED/workspaces/$TASK13_WORKSPACE_ID.yaml"
printf 'legacy guide\n' >"$TASK13_SEED/workspace-content/$TASK13_WORKSPACE_ID/evidence/guide.md"
[[ -f "$TASK13_SEED/workspaces/$TASK13_WORKSPACE_ID.yaml" ]] || task13_fail "missing Task 13 legacy flat descriptor fixture"
[[ -f "$TASK13_SEED/workspace-content/$TASK13_WORKSPACE_ID/evidence/guide.md" ]] || task13_fail "missing Task 13 legacy flat evidence fixture"
printf 'Task 13 fixture contract passed.
'
}
task13_initialize() {
umask 077
TASK13_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)"
TASK13_TMP_PARENT="$(cd "${TMPDIR:-/tmp}" && pwd -P)"
TASK13_TMP="$(mktemp -d "$TASK13_TMP_PARENT/thothii-task13.XXXXXX")"
TASK13_TMP="$(cd "$TASK13_TMP" && pwd -P)"
TASK13_LOG="$TASK13_TMP/task13.log"
TASK13_FAILURE_LOGGED=0
: >"$TASK13_LOG"
trap 'task13_cleanup $?' EXIT
trap 'exit 130' INT TERM HUP
TASK13_RUN_ID="$(date -u +%Y%m%d%H%M%S)-$$-${RANDOM:-0}"
TASK13_PROFILE="local"
TASK13_INSTALLATION="$TASK13_TMP/thothii-installation.yaml"
TASK13_PROJECT="thothii-$(task13_sha256_text "$TASK13_INSTALLATION" | cut -c1-12)"
TASK13_CONTROL_DIR="$TASK13_ROOT/.tht/$TASK13_PROJECT"
[[ ! -e "$TASK13_CONTROL_DIR" ]] || task13_fail "unique tht control directory already exists"
TASK13_CURRENT_IMAGE_OVERRIDE="$TASK13_CONTROL_DIR/current-image.yaml"
TASK13_UPDATE_STATE="$TASK13_CONTROL_DIR/update-state.json"
TASK13_REMOTE="$TASK13_TMP/remote.git"
TASK13_SEED="$TASK13_TMP/seed"
TASK13_BRANCH="task13-smoke"
TASK13_ENV_FILE="$TASK13_TMP/local.env"
TASK13_OVERRIDE="$TASK13_TMP/compose.task13.yaml"
TASK13_PI_AUTH="$TASK13_TMP/pi-auth.json"
TASK13_SECRETS="$TASK13_TMP/thothii.secrets"
TASK13_AUTH_ROOT="$TASK13_TMP/auth"
TASK13_AUTH_PASSWORD_FILE="$TASK13_TMP/local-auth-password"
TASK13_AUTH_ADMIN=task13-admin
TASK13_AUTH_PASSWORD="task13-auth-$TASK13_RUN_ID"
TASK13_OIDC_CLIENT_SECRET="task13-oidc-client-$TASK13_RUN_ID"
TASK13_AUTHENTIK_API_TOKEN="task13-authentik-token-$TASK13_RUN_ID"
TASK13_PI_MODELS="$TASK13_TMP/models.json"
TASK13_PI_SETTINGS="$TASK13_TMP/pi-settings.json"
TASK13_LLM_SERVER="$TASK13_TMP/fake-llm.mjs"
TASK13_OIDC_SERVER="$TASK13_TMP/fake-oidc.mjs"
TASK13_OIDC_CERT="$TASK13_TMP/fake-oidc-cert.pem"
TASK13_OIDC_KEY="$TASK13_TMP/fake-oidc-key.pem"
TASK13_THT_DIR="$TASK13_TMP/tht"
TASK13_LLM_CONTAINER="$TASK13_PROJECT-llm"
TASK13_OIDC_CONTAINER="$TASK13_PROJECT-oidc"
TASK13_BAD_CANDIDATE_CONTAINER="$TASK13_PROJECT-bad-candidate"
TASK13_CORE_IMAGE="task13-core-$TASK13_RUN_ID:local"
TASK13_FRONTEND_IMAGE="task13-frontend-$TASK13_RUN_ID:local"
TASK13_SECRET_VALUE="task13-secret-$TASK13_RUN_ID"
TASK13_NETWORK=""
TASK13_BAD_CANDIDATE_ID=""
TASK13_PREVIOUS_IMAGE_ID=""
TASK13_SERVER_DATA="$TASK13_TMP/Server Data"
TASK13_SERVER_PI_STATE="$TASK13_TMP/Server Pi State"
TASK13_SERVER_REGISTRY="$TASK13_TMP/Server Registry"
TASK13_SERVER_WORKSPACE_CONFIG="$TASK13_TMP/server-sessions.yaml"
TASK13_SESSION_RUNTIME_PASSWORD="$TASK13_TMP/session-runtime-password"
TASK13_SESSION_MIGRATOR_PASSWORD_FILE="$TASK13_TMP/session-migrator-password"
TASK13_SESSION_CA="$TASK13_TMP/session-ca.pem"
TASK13_SESSION_PASSWORD="task13-runtime-$TASK13_RUN_ID"
TASK13_SESSION_MIGRATOR_PASSWORD="task13-migrator-$TASK13_RUN_ID"
TASK13_FRONTEND_PORT=""
}
task13_require_tools() {
for command in bash git docker curl node openssl python3 sed awk grep rg sort; do
command -v "$command" >/dev/null 2>&1 || task13_fail "$command is required"
done
task13_run_logged "Docker daemon readiness" docker info
task13_run_logged "Docker Compose readiness" docker compose version
task13_self_test_source_contract
}
task13_smoke_main() {
local mode="${1:-full}"
[[ "$mode" == full || "$mode" == update ]] || task13_fail "unknown Task 13 smoke mode: $mode"
task13_initialize
task13_require_tools
TASK13_FRONTEND_PORT="$(node -e 'const net=require("node:net"); const server=net.createServer(); server.listen(0,"127.0.0.1",()=>{process.stdout.write(String(server.address().port)); server.close()})')"
[[ "$TASK13_FRONTEND_PORT" =~ ^[1-9][0-9]*$ ]] || task13_fail "could not reserve a loopback frontend port"
task13_write_fixture_files
task13_write_environment /fixtures/remote.git
task13_seed_registry
task13_build_tht
task13_configure_local_authentication
task13_start_stack
task13_assert_project_ownership
task13_assert_built_image_ownership
task13_assert_runtime
task13_assert_local_restore_reauthentication
task13_prepare_persistence
if [[ "$mode" == full ]]; then
task13_registry_lifecycle
fi
task13_update_rollback
printf 'Task 13 %s deployment smoke passed.\n' "$mode"
}
task13_server_smoke_main() {
task13_initialize
TASK13_PROFILE="server"
task13_require_tools
task13_write_server_fixture_files
task13_seed_registry
task13_build_tht
task13_configure_server_oidc_authentication
task13_start_server_stack
task13_assert_project_ownership
task13_assert_built_image_ownership
task13_assert_server_runtime
printf 'Task 13 Linux server deployment smoke passed.\n'
}
if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
if [[ "${1:-}" == "--self-test" ]]; then
task13_self_test
elif [[ "${1:-}" == "--self-test-case" ]]; then
[[ -n "${2:-}" ]] || task13_fail "--self-test-case requires a case name"
task13_self_test_case "$2"
elif [[ "${1:-}" == "--fixtures-only" ]]; then
task13_fixtures_only
else
task13_supervise "$TASK13_SMOKE_TIMEOUT" "unified deployment smoke" task13_smoke_main full
fi
fi
Reference in New Issue View Git Blame Copy Permalink