#!/usr/bin/env bash # End-to-end release gate for the canonical two-service Compose distribution. # This file is also sourced by tht-update-smoke.sh so both entry points use the same # isolated fixture, exact cleanup, and sanitized failure reporting. set -euo pipefail TASK13_BAD_CANDIDATE_IMAGE="hello-world@sha256:c3cbe1cc1aa588a64951ac6286e0df7b27fe2e6324b1001c619bb358770c0178" TASK13_BAD_CANDIDATE_BEHAVIOR="stopped" TASK13_BAD_PI_VERSION="0.80.4-task13" TASK13_CURL_CONNECT_TIMEOUT=3 TASK13_CURL_MAX_TIME=10 TASK13_CLEANUP_TIMEOUT=20 TASK13_COMMAND_TIMEOUT=900 TASK13_SMOKE_TIMEOUT=1800 TASK13_TERM_GRACE=45 task13_fail() { printf 'Task 13 smoke failed: %s\n' "$*" >&2 return 1 } task13_sha256_text() { if command -v sha256sum >/dev/null 2>&1; then printf '%s' "$1" | sha256sum | awk '{print $1}' elif command -v shasum >/dev/null 2>&1; then printf '%s' "$1" | shasum -a 256 | awk '{print $1}' else task13_fail "sha256sum or shasum is required" fi } task13_record_image_evidence() { local reference="$1" role="$2" image_id verified_id repo_digests [[ "$role" =~ ^[a-z0-9-]+$ ]] || task13_fail "invalid image evidence role" image_id="$(docker image inspect --format '{{.Id}}' "$reference")" [[ "$image_id" =~ ^sha256:[0-9a-f]{64}$ ]] \ || task13_fail "Docker image identity was not resolved" verified_id="$(docker image inspect --format '{{.Id}}' "$image_id")" [[ "$verified_id" == "$image_id" ]] \ || task13_fail "Docker image identity changed during evidence capture" if ! repo_digests="$({ docker image inspect --format '{{json .RepoDigests}}' "$image_id"; } | node -e ' const fs = require("node:fs"); let value = JSON.parse(fs.readFileSync(0, "utf8")); if (value === null) value = []; if (!Array.isArray(value)) process.exit(1); const digests = []; for (const item of value) { if (typeof item !== "string" || !/@sha256:[0-9a-f]{64}$/.test(item)) process.exit(1); digests.push(item.slice(item.lastIndexOf("@") + 1)); } process.stdout.write(JSON.stringify([...new Set(digests)].sort())); ')"; then task13_fail "Docker repository digest evidence was invalid" return 1 fi printf '%s\t%s\t%s\n' "$image_id" "$role" "$repo_digests" >>"$TASK13_IMAGE_EVIDENCE_RECORDS" } task13_record_project_image_evidence() { local container_id image_id count=0 while IFS= read -r container_id; do [[ -n "$container_id" ]] || continue image_id="$(docker container inspect --format '{{.Image}}' "$container_id")" task13_record_image_evidence "$image_id" compose-runtime count=$((count + 1)) done < <(docker container ls -aq --filter "label=com.docker.compose.project=$TASK13_PROJECT") [[ "$count" -gt 0 ]] || task13_fail "no Compose runtime images were available for evidence capture" } task13_write_image_evidence() { local image_id verified_id output_dir temporary while IFS= read -r image_id; do [[ -n "$image_id" ]] || continue verified_id="$(docker image inspect --format '{{.Id}}' "$image_id")" [[ "$verified_id" == "$image_id" ]] \ || task13_fail "Docker image identity was unavailable before cleanup" done < <(cut -f1 "$TASK13_IMAGE_EVIDENCE_RECORDS" | LC_ALL=C sort -u) output_dir="$(dirname "$TASK13_IMAGE_EVIDENCE_OUTPUT")" mkdir -p "$output_dir" temporary="$(mktemp "$output_dir/.unified-docker-images.XXXXXX")" if ! node - "$TASK13_IMAGE_EVIDENCE_RECORDS" "$TASK13_SOURCE_COMMIT" "$TASK13_RUN_ID" \ "$TASK13_IMAGE_EVIDENCE_STATUS" >"$temporary" <<'NODE' const fs = require("node:fs"); const [recordsFile, sourceCommit, runId, status] = process.argv.slice(2); if (!/^[0-9a-f]{40}$/.test(sourceCommit) || !/^[0-9A-Za-z-]+$/.test(runId) || !/^(?:pass|fail)$/.test(status)) process.exit(1); const images = new Map(); for (const line of fs.readFileSync(recordsFile, "utf8").split("\n").filter(Boolean)) { const fields = line.split("\t"); if (fields.length !== 3) process.exit(1); const [id, role, encodedDigests] = fields; if (!/^sha256:[0-9a-f]{64}$/.test(id) || !/^[a-z0-9-]+$/.test(role)) process.exit(1); const repoDigests = JSON.parse(encodedDigests); if (!Array.isArray(repoDigests) || repoDigests.some((digest) => typeof digest !== "string" || !/^sha256:[0-9a-f]{64}$/.test(digest))) { process.exit(1); } const current = images.get(id) ?? { id, roles: new Set(), repo_digests: new Set() }; current.roles.add(role); for (const digest of repoDigests) current.repo_digests.add(digest); images.set(id, current); } if (images.size === 0) process.exit(1); const document = { gate: "unified-deployment-smoke", status, source_commit: sourceCommit, run_id: runId, images: [...images.values()].sort((left, right) => left.id.localeCompare(right.id)).map((image) => ({ id: image.id, roles: [...image.roles].sort(), repo_digests: [...image.repo_digests].sort(), })), }; process.stdout.write(`${JSON.stringify(document, null, 2)}\n`); NODE then rm -f "$temporary" task13_fail "could not serialize sanitized Docker image evidence" return 1 fi chmod 0600 "$temporary" mv "$temporary" "$TASK13_IMAGE_EVIDENCE_OUTPUT" } task13_capture_failed_image_evidence() { TASK13_IMAGE_EVIDENCE_STATUS=fail task13_record_project_image_evidence task13_write_image_evidence } task13_registry_filesystem_fingerprint() { python3 - "$1" <<'PY' import hashlib import os import stat import sys root = os.path.abspath(sys.argv[1]) records = [] entries = 0 total_bytes = 0 def snapshot(value): return ( value.st_dev, value.st_ino, value.st_mode, value.st_uid, value.st_gid, value.st_size, value.st_mtime_ns, value.st_ctime_ns, ) def visit(path, relative): global entries, total_bytes before = os.lstat(path) entries += 1 if entries > 65536: raise SystemExit("registry fingerprint entry bound exceeded") metadata = snapshot(before) if stat.S_ISLNK(before.st_mode): raise SystemExit("registry fingerprint rejected a symbolic link") if stat.S_ISDIR(before.st_mode): records.append(("directory", relative, metadata)) with os.scandir(path) as listing: children = sorted((item.name for item in listing)) for name in children: visit(os.path.join(path, name), name if relative == "." else relative + "/" + name) elif stat.S_ISREG(before.st_mode): if before.st_size > 256 * 1024 * 1024: raise SystemExit("registry fingerprint file bound exceeded") total_bytes += before.st_size if total_bytes > 2 * 1024 * 1024 * 1024: raise SystemExit("registry fingerprint total bound exceeded") flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0) descriptor = os.open(path, flags) try: opened = os.fstat(descriptor) if snapshot(opened) != metadata: raise SystemExit("registry changed while fingerprinting") digest = hashlib.sha256() while True: chunk = os.read(descriptor, 1024 * 1024) if not chunk: break digest.update(chunk) finally: os.close(descriptor) records.append(("file", relative, metadata, digest.hexdigest())) else: raise SystemExit("registry fingerprint rejected a special file") if snapshot(os.lstat(path)) != metadata: raise SystemExit("registry changed while fingerprinting") visit(root, ".") encoded = repr(records).encode("utf-8") print("sha256:" + hashlib.sha256(encoded).hexdigest()) PY } task13_sanitize() { local line while IFS= read -r line || [[ -n "$line" ]]; do if [[ -n "${TASK13_SECRET_VALUE:-}" ]]; then line="${line//"$TASK13_SECRET_VALUE"/[REDACTED]}" fi printf '%s\n' "$line" done | sed -E \ -e 's#([[:alpha:]][[:alnum:]+.-]*://[^:/@[:space:]]+:)[^@/[:space:]]+@#\1[REDACTED]@#g' \ -e 's/(([Pp]roxy-)?[Aa]uthorization:[[:space:]]*([Bb]earer|[Bb]asic)[[:space:]]+)[^[:space:]]+/\1[REDACTED]/g' \ -e "s/(([\"']?[[:alnum:]_.-]*(password|token|api[_-]?key|secret|key)[[:alnum:]_.-]*[\"']?[[:space:]]*[:=][[:space:]]*)([\"'][^\"']*[\"']|[^[:space:],;]+))/\2[REDACTED]/Ig" } task13_log_failure() { local label="$1" TASK13_FAILURE_LOGGED=1 printf 'Task 13 command failed: %s\n' "$label" >&2 tail -n 200 "$TASK13_LOG" | task13_sanitize >&2 return 1 } task13_run_logged() { local label="$1" shift if ! task13_bounded "$TASK13_COMMAND_TIMEOUT" "$label" "$@" >>"$TASK13_LOG" 2>&1; then task13_log_failure "$label" fi } task13_bounded() { local seconds="$1" label="$2" command_pid watchdog_pid rc watchdog_rc=0 local monitor_enabled=0 timeout_marker command_group shift 2 timeout_marker="$(mktemp "${TMPDIR:-/tmp}/thothii-task13-timeout.XXXXXX")" [[ $- == *m* ]] && monitor_enabled=1 if [[ -n "${TASK13_ACTIVE_GROUP:-}" ]]; then [[ "$TASK13_ACTIVE_GROUP" =~ ^[0-9]+$ ]] \ || task13_fail "invalid active Task 13 process group" set +m "$@" & command_pid=$! command_group="$TASK13_ACTIVE_GROUP" else set -m "$@" & command_pid=$! command_group="$command_pid" [[ "$monitor_enabled" -eq 1 ]] || set +m fi ( local sleep_pid grace_deadline sleep "$seconds" & sleep_pid=$! trap 'kill "$sleep_pid" 2>/dev/null || true' EXIT INT TERM wait "$sleep_pid" 2>/dev/null || exit 0 trap - EXIT INT TERM if kill -0 -- "-$command_group" 2>/dev/null; then trap '' TERM printf 'timeout\n' >"$timeout_marker" printf 'Task 13 command timed out after %ss: %s\n' "$seconds" "$label" >&2 kill -TERM -- "-$command_group" 2>/dev/null || true grace_deadline=$((SECONDS + TASK13_TERM_GRACE)) while kill -0 -- "-$command_group" 2>/dev/null && ((SECONDS < grace_deadline)); do sleep 1 done kill -KILL -- "-$command_group" 2>/dev/null || true exit 124 fi ) & watchdog_pid=$! if [[ -n "${TASK13_ACTIVE_GROUP:-}" && "$monitor_enabled" -eq 1 ]]; then set -m fi if wait "$command_pid"; then rc=0 else rc=$? fi if [[ -s "$timeout_marker" ]]; then wait "$watchdog_pid" 2>/dev/null || watchdog_rc=$? else kill "$watchdog_pid" 2>/dev/null || true wait "$watchdog_pid" 2>/dev/null || true fi rm -f "$timeout_marker" [[ "$watchdog_rc" -eq 124 ]] && return 124 return "$rc" } task13_supervised_call() { TASK13_ACTIVE_GROUP="$BASHPID" "$@" } task13_supervise() { local seconds="$1" label="$2" shift 2 task13_bounded "$seconds" "$label" task13_supervised_call "$@" } task13_compose_files() { TASK13_COMPOSE=( docker compose --project-name "$TASK13_PROJECT" --project-directory "$TASK13_ROOT" --env-file "$TASK13_ENV_FILE" -f "$TASK13_ROOT/compose.yaml" ) if [[ "${TASK13_PROFILE:-local}" == server ]]; then TASK13_COMPOSE+=( -f "$TASK13_ROOT/deploy/compose.server.yaml" -f "$TASK13_ROOT/deploy/compose.session-server.yaml.example" -f "$TASK13_OVERRIDE" -f "$TASK13_ROOT/deploy/compose.auth-runtime-projection.yaml" ) else TASK13_COMPOSE+=( -f "$TASK13_ROOT/deploy/compose.local.yaml" -f "$TASK13_OVERRIDE" ) fi if [[ "${TASK13_PROFILE:-local}" == local && -f "$TASK13_CURRENT_IMAGE_OVERRIDE" ]]; then TASK13_COMPOSE+=(-f "$TASK13_CURRENT_IMAGE_OVERRIDE") fi } task13_compose() { task13_compose_files "${TASK13_COMPOSE[@]}" "$@" } task13_compose_logged() { local label="$1" shift task13_compose_files task13_run_logged "$label" "${TASK13_COMPOSE[@]}" "$@" } task13_report_core_startup_failure() { local container_id state exit_code logs cause="startup failure is unclassified" container_id="$(task13_compose ps --all -q core 2>/dev/null | head -n 1 || true)" state="" if [[ -n "$container_id" ]]; then state="$(docker inspect --format '{{.State.Status}}:{{.State.ExitCode}}' "$container_id" 2>/dev/null || true)" fi exit_code="${state##*:}" [[ "$exit_code" =~ ^[0-9]{1,3}$ ]] || exit_code="unknown" logs="$(task13_compose logs --no-color --tail 100 core 2>/dev/null || true)" case "$logs" in *auth_session_store_invalid*|*auth*storage*request*failed*|*EACCES*auth*|*permission*auth*) cause="authentication state storage is unavailable" ;; *auth_config_invalid*|*authentication*configuration*) cause="authentication configuration is invalid" ;; *workspace_registry_invalid*|*workspace*registry*) cause="workspace registry startup validation failed" ;; esac printf 'Core startup cause: %s (exit code %s).\n' "$cause" "$exit_code" >&2 } task13_compose_start_logged() { local label="$1" shift task13_compose_files if task13_bounded "$TASK13_COMMAND_TIMEOUT" "$label" \ "${TASK13_COMPOSE[@]}" "$@" >>"$TASK13_LOG" 2>&1; then return 0 fi TASK13_FAILURE_LOGGED=1 printf 'Task 13 command failed: %s\n' "$label" >&2 task13_report_core_startup_failure return 1 } task13_write_environment() { local remote="$1" { printf 'THOTH_HTTP_PORT=%s\n' "$TASK13_FRONTEND_PORT" printf 'THOTH_CORE_HTTP_PORT=0\n' printf 'MAX_PI_PROCESSES=2\n' printf 'PI_AUTH_FILE=%s\n' "$TASK13_PI_AUTH" printf 'THT_SECRETS_FILE=%s\n' "$TASK13_SECRETS" printf 'THT_AUTH_CONFIG_ROOT=%s\n' "$TASK13_AUTH_ROOT" printf 'THT_WORKSPACE_GIT_REMOTE=%s\n' "$remote" printf 'THT_WORKSPACE_GIT_BRANCH=%s\n' "$TASK13_BRANCH" printf 'THT_LLM_URL=http://%s:9000/v1\n' "$TASK13_LLM_CONTAINER" } >"$TASK13_ENV_FILE" chmod 0600 "$TASK13_ENV_FILE" } task13_write_session_ca_fixture() { cat >"$TASK13_SESSION_CA" <<'EOF' -----BEGIN CERTIFICATE----- VEFTSzEzLURJU1BPU0FCTEUtU0VTU0lPTi1DQQ== -----END CERTIFICATE----- EOF chmod 0600 "$TASK13_SESSION_CA" } task13_write_fixture_files() { printf '{}\n' >"$TASK13_PI_AUTH" printf 'THT_MODEL_API_KEY=%s\n' "$TASK13_SECRET_VALUE" >"$TASK13_SECRETS" printf '%s' "task13-runtime-password-$TASK13_RUN_ID" >"$TASK13_SESSION_RUNTIME_PASSWORD" printf '%s' "$TASK13_AUTH_PASSWORD" >"$TASK13_AUTH_PASSWORD_FILE" task13_write_session_ca_fixture mkdir -p "$TASK13_AUTH_ROOT" chmod 0600 "$TASK13_PI_AUTH" chmod 0700 "$TASK13_AUTH_ROOT" chmod 0600 "$TASK13_SECRETS" "$TASK13_SESSION_RUNTIME_PASSWORD" "$TASK13_AUTH_PASSWORD_FILE" cat >"$TASK13_PI_MODELS" <"$TASK13_PI_SETTINGS" <<'EOF' { "defaultProjectTrust": "always", "enabledModels": ["local-qwen/task13-smoke"] } EOF chmod 0644 "$TASK13_PI_MODELS" "$TASK13_PI_SETTINGS" cat >"$TASK13_LLM_SERVER" <<'EOF' import http from "node:http"; const server = http.createServer((request, response) => { if (request.method === "GET" && request.url === "/health") { response.writeHead(200, { "content-type": "application/json" }); response.end('{"status":"ok"}'); return; } if (request.method === "GET" && request.url === "/v1/models") { response.writeHead(200, { "content-type": "application/json" }); response.end('{"object":"list","data":[{"id":"task13-smoke","object":"model"}]}'); return; } if (request.method !== "POST" || request.url !== "/v1/chat/completions") { response.writeHead(404, { "content-type": "application/json" }); response.end('{"error":{"message":"not found"}}'); return; } let body = ""; request.setEncoding("utf8"); request.on("data", (chunk) => { body += chunk; }); request.on("end", () => { let stream = true; try { stream = JSON.parse(body).stream !== false; } catch { /* return the safe fixture */ } if (!stream) { response.writeHead(200, { "content-type": "application/json" }); response.end(JSON.stringify({ id: "task13", object: "chat.completion", created: 1, model: "task13-smoke", choices: [{ index: 0, message: { role: "assistant", content: "OK" }, finish_reason: "stop" }], })); return; } response.writeHead(200, { "content-type": "text/event-stream", "cache-control": "no-cache", connection: "keep-alive", }); response.write('data: {"id":"task13","object":"chat.completion.chunk","created":1,"model":"task13-smoke","choices":[{"index":0,"delta":{"role":"assistant","content":"OK"},"finish_reason":null}]}\n\n'); response.write('data: {"id":"task13","object":"chat.completion.chunk","created":1,"model":"task13-smoke","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}\n\n'); response.end("data: [DONE]\n\n"); }); }); server.listen(9000, "0.0.0.0"); EOF chmod 0644 "$TASK13_LLM_SERVER" cat >"$TASK13_OVERRIDE" <"$TASK13_INSTALLATION" <"$TASK13_PI_AUTH" printf 'THT_MODEL_API_KEY=%s\nTHT_OIDC_CLIENT_SECRET=%s\nTHT_AUTHENTIK_API_TOKEN=%s\n' \ "$TASK13_SECRET_VALUE" "$TASK13_OIDC_CLIENT_SECRET" "$TASK13_AUTHENTIK_API_TOKEN" >"$TASK13_SECRETS" printf '%s' "$TASK13_SESSION_PASSWORD" >"$TASK13_SESSION_RUNTIME_PASSWORD" printf '%s' "$TASK13_SESSION_MIGRATOR_PASSWORD" >"$TASK13_SESSION_MIGRATOR_PASSWORD_FILE" task13_write_session_ca_fixture chmod 0600 "$TASK13_PI_AUTH" chmod 0600 "$TASK13_SECRETS" "$TASK13_SESSION_RUNTIME_PASSWORD" \ "$TASK13_SESSION_MIGRATOR_PASSWORD_FILE" task13_run_logged "create scoped fake OIDC certificate" openssl req -x509 -newkey rsa:2048 \ -sha256 -nodes -days 1 -subj '/CN=task13-fake-oidc' \ -addext 'subjectAltName=DNS:task13-fake-oidc' \ -keyout "$TASK13_OIDC_KEY" -out "$TASK13_OIDC_CERT" chmod 0600 "$TASK13_OIDC_KEY" chmod 0644 "$TASK13_OIDC_CERT" cat >"$TASK13_OIDC_SERVER" <<'EOF' import { generateKeyPairSync } from "node:crypto"; import { readFileSync } from "node:fs"; import https from "node:https"; const origin = "https://task13-fake-oidc:9443"; const issuer = `${origin}/application/o/task13/`; const expectedToken = process.env.TASK13_AUTHENTIK_API_TOKEN; const { publicKey } = generateKeyPairSync("rsa", { modulusLength: 2048 }); const jwk = { ...publicKey.export({ format: "jwk" }), kid: "task13", use: "sig", alg: "RS256" }; const send = (response, status, body) => { const payload = JSON.stringify(body); response.writeHead(status, { "content-type": "application/json", "content-length": Buffer.byteLength(payload) }); response.end(payload); }; const server = https.createServer({ cert: readFileSync("/fixtures/oidc-cert.pem"), key: readFileSync("/fixtures/oidc-key.pem"), }, (request, response) => { const target = new URL(request.url ?? "/", origin); if (target.pathname === "/health") return send(response, 200, { status: "ok" }); if (target.pathname.includes(".well-known/openid-configuration")) { return send(response, 200, { issuer, authorization_endpoint: `${origin}/authorize`, token_endpoint: `${origin}/token`, jwks_uri: `${origin}/jwks`, response_types_supported: ["code"], subject_types_supported: ["public"], id_token_signing_alg_values_supported: ["RS256"], }); } if (target.pathname === "/jwks") return send(response, 200, { keys: [jwk] }); if (target.pathname === "/api/v3/core/groups/") { if (request.headers.authorization !== `Bearer ${expectedToken}`) return send(response, 401, { detail: "unauthorized" }); const name = target.searchParams.get("name") ?? ""; console.log(`group:${name}`); const configured = name === "task13-users" || name === "task13-admins"; return send(response, 200, { pagination: { next: null }, results: configured ? [{ name }, { name: "task13-unrelated" }] : [{ name: "task13-unrelated" }], }); } return send(response, 404, { error: "not_found" }); }); server.listen(9443, "0.0.0.0"); EOF chmod 0644 "$TASK13_OIDC_SERVER" cat >"$TASK13_SERVER_WORKSPACE_CONFIG" <<'EOF' language: en session_storage: type: postgres_direct connection: host: ${THT_SESSION_DB_HOST} port: ${THT_SESSION_DB_PORT} database: ${THT_SESSION_DB_NAME} schema: thoth_sessions user: ${THT_SESSION_RUNTIME_USER} password_file: ${THT_SESSION_RUNTIME_PASSWORD_FILE} sslmode: ${THT_SESSION_DB_SSLMODE} sslrootcert: ${THT_SESSION_DB_SSLROOTCERT} roots: artifacts: artifacts indexes: indexes sessions: sessions EOF chmod 0644 "$TASK13_SERVER_WORKSPACE_CONFIG" mkdir -p "$TASK13_SERVER_DATA" "$TASK13_SERVER_PI_STATE" "$TASK13_SERVER_REGISTRY" "$TASK13_ROOT/scripts/prepare-server-pi-state.sh" \ "$TASK13_SERVER_PI_STATE" "$(id -u)" "$(id -g)" >>"$TASK13_LOG" chmod 0777 "$TASK13_SERVER_DATA" "$TASK13_SERVER_PI_STATE" \ "$TASK13_SERVER_PI_STATE/agent" "$TASK13_SERVER_REGISTRY" data_root="$TASK13_SERVER_DATA" pi_root="$TASK13_SERVER_PI_STATE" registry_root="$TASK13_SERVER_REGISTRY" remote_path="$TASK13_REMOTE" workspace_path="$TASK13_SERVER_WORKSPACE_CONFIG" cat >"$TASK13_OVERRIDE" <"$TASK13_ENV_FILE" chmod 0600 "$TASK13_ENV_FILE" cat >"$TASK13_INSTALLATION" <"$catalog_path" } task13_replace_once() { local target="$1" old="$2" new="$3" python3 - "$target" "$old" "$new" <<'PY' from pathlib import Path import sys path = Path(sys.argv[1]) old = sys.argv[2] new = sys.argv[3] text = path.read_text() count = text.count(old) if count != 1: raise SystemExit(f"expected exactly one occurrence of {old!r} in {path}, found {count}") path.write_text(text.replace(old, new, 1)) PY } task13_commit_registry_change() { local message="$1" task13_run_logged "$message" git -C "$TASK13_SEED" add -A task13_run_logged "$message" git -C "$TASK13_SEED" -c user.name='Task 13 Smoke' -c user.email='task13-smoke@example.invalid' commit -m "$message" task13_run_logged "$message" git -C "$TASK13_SEED" push "$TASK13_REMOTE" "HEAD:$TASK13_BRANCH" if [[ -n "${TASK13_REGISTRY_RUNTIME_VOLUME:-}" ]] \ && docker volume inspect "$TASK13_REGISTRY_RUNTIME_VOLUME" >/dev/null 2>&1; then task13_prepare_registry_remote fi } task13_seed_registry() { local fixture metadata fixture="$TASK13_ROOT/scripts/fixtures/workspace-registry-task13.yaml" mapfile -t metadata < <(task13_workspace_metadata "$fixture") TASK13_WORKSPACE_ID="${metadata[0]}" TASK13_WORKSPACE_NAME="${metadata[1]}" TASK13_WORKSPACE_DESCRIPTION="${metadata[2]-}" mkdir -p "$TASK13_SEED" task13_run_logged "initialize bare workspace registry" git init --bare --initial-branch=main "$TASK13_REMOTE" task13_run_logged "initialize workspace seed" git -C "$TASK13_SEED" init --initial-branch=main mkdir -p "$TASK13_SEED/$TASK13_WORKSPACE_ID" cp "$fixture" "$TASK13_SEED/$TASK13_WORKSPACE_ID/workspace.yaml" task13_write_catalog "$TASK13_SEED/thoth-workspaces.yaml" "$TASK13_WORKSPACE_ID" "$TASK13_WORKSPACE_NAME" "$TASK13_WORKSPACE_DESCRIPTION" if grep -Fq 'type: filesystem' "$fixture"; then mkdir -p "$TASK13_SEED/$TASK13_WORKSPACE_ID/evidence" printf 'guide v1\n' >"$TASK13_SEED/$TASK13_WORKSPACE_ID/evidence/guide.md" fi task13_commit_registry_change 'Seed Task 13 workspace registry' } task13_build_tht() { local os arch mkdir -p "$TASK13_THT_DIR" task13_run_logged "build tht cross-platform binaries" env \ THT_THT_OUTPUT_DIRECTORY="$TASK13_THT_DIR" \ bash "$TASK13_ROOT/scripts/build-tht.sh" os="$(uname -s)" arch="$(uname -m)" case "$os/$arch" in Darwin/x86_64) TASK13_THT="$TASK13_THT_DIR/tht-darwin-amd64" ;; Darwin/arm64) TASK13_THT="$TASK13_THT_DIR/tht-darwin-arm64" ;; Linux/x86_64|Linux/amd64) TASK13_THT="$TASK13_THT_DIR/tht-linux-amd64" ;; Linux/aarch64|Linux/arm64) TASK13_THT="$TASK13_THT_DIR/tht-linux-arm64" ;; *) task13_fail "unsupported smoke host: $os/$arch" ;; esac chmod 0700 "$TASK13_THT" task13_run_logged "invoke host tht" "$TASK13_THT" --help } task13_assert_rendered_contract() { local services rendered services="$(task13_compose config --services | sort)" [[ "$services" == $'core\nembedding\nembedding-model-init\nfrontend\nqdrant' ]] \ || task13_fail "rendered stack is not the mandatory internal semantic topology" rendered="$TASK13_TMP/rendered-compose.yaml" task13_compose config >"$rendered" if grep -Eqi 'docker\.sock|/var/run/docker' "$rendered"; then task13_fail "rendered Compose exposes a Docker daemon endpoint" fi if grep -Fq "$TASK13_SECRET_VALUE" "$rendered"; then task13_fail "rendered Compose exposed the fixture secret" fi for endpoint in THT_DWH_REST_URL THT_LLM_URL \ THT_INTERNAL_QDRANT_URL THT_INTERNAL_EMBEDDING_URL \ THT_INTERNAL_EMBEDDING_MODEL THT_INTERNAL_EMBEDDING_DIMENSIONS; do grep -Fq "$endpoint" "$rendered" || task13_fail "rendered Compose lacks $endpoint" done if grep -Eq 'THT_VEC_REST_URL|THT_VEC_WRITE_REST_URL|THT_OLLAMA_URL' "$rendered"; then task13_fail "rendered Compose still exposes retired external semantic bindings" fi grep -Fq '/run/thothii-auth' "$rendered" || task13_fail "rendered Compose lacks the read-only auth configuration mount" grep -Fq '/data/auth' "$rendered" || task13_fail "rendered Compose lacks authentication state storage" } task13_configure_local_authentication() { task13_run_logged "configure local authentication" "$TASK13_THT" --installation "$TASK13_INSTALLATION" auth configure \ --mode local --public-url "http://127.0.0.1:$TASK13_FRONTEND_PORT" \ --admin-user "$TASK13_AUTH_ADMIN" --admin-display-name "Task 13 Administrator" \ --password-file "$TASK13_AUTH_PASSWORD_FILE" } task13_configure_server_oidc_authentication() { task13_run_logged "configure fake server OIDC authentication" task13_server_tht auth configure \ --mode oidc --public-url "https://task13.example.invalid" \ --issuer "https://task13-fake-oidc:9443/application/o/task13/" --client-id task13-smoke-client \ --authentik-base-url "https://task13-fake-oidc:9443" --user-group task13-users --admin-group task13-admins } task13_prepare_server_auth_roots() { [[ "${TASK13_PROFILE:-}" == server \ && "$TASK13_AUTH_ROOT" == "$TASK13_TMP/auth" \ && ! -L "$TASK13_AUTH_ROOT" \ && ( ! -e "$TASK13_AUTH_ROOT" || -d "$TASK13_AUTH_ROOT" ) \ && "$TASK13_AUTH_RUNTIME_ROOT" == "$TASK13_TMP/auth-runtime" \ && ! -L "$TASK13_AUTH_RUNTIME_ROOT" \ && ( ! -e "$TASK13_AUTH_RUNTIME_ROOT" || -d "$TASK13_AUTH_RUNTIME_ROOT" ) ]] \ || task13_fail "refusing to prepare unexpected server authentication roots" task13_run_logged "prepare server authentication canonical root" sudo -n -- \ install -d -o 0 -g 0 -m 0700 -- "$TASK13_AUTH_ROOT" task13_run_logged "prepare server authentication runtime root" sudo -n -- \ install -d -o 10001 -g 10001 -m 0700 -- "$TASK13_AUTH_RUNTIME_ROOT" } task13_prepare_server_secret_sources() { local path [[ "${TASK13_PROFILE:-}" == server && -n "${TASK13_TMP:-}" ]] \ || task13_fail "refusing to prepare server secret sources outside the server fixture" for path in \ "$TASK13_SECRETS" \ "$TASK13_PI_AUTH" \ "$TASK13_SESSION_RUNTIME_PASSWORD" \ "$TASK13_SESSION_MIGRATOR_PASSWORD_FILE" \ "$TASK13_SESSION_CA"; do [[ "$path" == "$TASK13_TMP/"* && -f "$path" && ! -L "$path" ]] \ || task13_fail "refusing to prepare an unexpected server secret source" done task13_run_logged "assign server secret sources to the container UID" sudo -n -- \ chown 10001:10001 -- "$TASK13_SECRETS" "$TASK13_PI_AUTH" \ "$TASK13_SESSION_RUNTIME_PASSWORD" "$TASK13_SESSION_MIGRATOR_PASSWORD_FILE" \ "$TASK13_SESSION_CA" task13_run_logged "protect server secret sources" sudo -n -- chmod 0600 -- \ "$TASK13_SECRETS" "$TASK13_PI_AUTH" "$TASK13_SESSION_RUNTIME_PASSWORD" \ "$TASK13_SESSION_MIGRATOR_PASSWORD_FILE" "$TASK13_SESSION_CA" } task13_server_tht() { sudo -n -- "$TASK13_THT" --installation "$TASK13_INSTALLATION" "$@" } task13_server_checkpoint_leftover() { [[ "${TASK13_PROFILE:-local}" == server \ && -n "${TASK13_CONTROL_DIR:-}" \ && "$TASK13_CONTROL_DIR" == "$TASK13_ROOT/.tht/$TASK13_PROJECT" \ && "$TASK13_PROJECT" =~ ^thothii-[0-9a-f]{12}$ ]] \ || task13_fail "refusing to inspect an unexpected server control path" sudo -n -- find "$TASK13_CONTROL_DIR" -maxdepth 1 -name 'restore-checkpoint-*.zip' -print -quit } task13_prepare_local_auth_runtime() { local owner_label owner_label="$(docker volume inspect --format '{{ index .Labels "io.thothii.task13.run" }}' \ "$TASK13_AUTH_RUNTIME_VOLUME")" [[ "$owner_label" == "$TASK13_RUN_ID" ]] \ || task13_fail "local authentication runtime volume lacks the Task 13 run label" task13_run_logged "project local authentication for the core runtime" docker run --rm \ --name "$TASK13_AUTH_PROJECTION_CONTAINER" \ --label "io.thothii.task13.run=$TASK13_RUN_ID" \ --user 0:0 \ --entrypoint sh \ --volume "$TASK13_AUTH_ROOT:/source:ro" \ --volume "$TASK13_AUTH_RUNTIME_VOLUME:/target" \ "$TASK13_CORE_IMAGE" -ceu ' test -f /source/auth.yaml && test ! -L /source/auth.yaml test -f /source/users.yaml && test ! -L /source/users.yaml test -z "$(find /target -mindepth 1 -maxdepth 1 -print -quit)" cp /source/auth.yaml /source/users.yaml /target/ chown 10001:10001 /target /target/auth.yaml /target/users.yaml chmod 0700 /target chmod 0600 /target/auth.yaml /target/users.yaml test "$(stat -c "%u:%g:%a" /target)" = 10001:10001:700 test "$(stat -c "%u:%g:%a" /target/auth.yaml)" = 10001:10001:600 test "$(stat -c "%u:%g:%a" /target/users.yaml)" = 10001:10001:600 ' } task13_prepare_local_pi_runtime() { local owner_label owner_label="$(docker volume inspect --format '{{ index .Labels "io.thothii.task13.run" }}' \ "$TASK13_PI_RUNTIME_VOLUME")" [[ "$owner_label" == "$TASK13_RUN_ID" ]] \ || task13_fail "local Pi runtime volume lacks the Task 13 run label" task13_run_logged "project local Pi configuration for the core runtime" docker run --rm \ --name "$TASK13_PI_PROJECTION_CONTAINER" \ --label "io.thothii.task13.run=$TASK13_RUN_ID" \ --user 0:0 \ --entrypoint sh \ --volume "$TASK13_PI_AUTH:/source/auth.json:ro" \ --volume "$TASK13_PI_MODELS:/source/models.json:ro" \ --volume "$TASK13_PI_SETTINGS:/source/settings.json:ro" \ --volume "$TASK13_PI_RUNTIME_VOLUME:/target" \ "$TASK13_CORE_IMAGE" -ceu ' test -f /source/auth.json && test ! -L /source/auth.json test -f /source/models.json && test ! -L /source/models.json test -f /source/settings.json && test ! -L /source/settings.json test -d /target/agent && test ! -L /target/agent test -z "$(find /target -mindepth 1 -maxdepth 1 ! -name agent -print -quit)" test -z "$(find /target/agent -mindepth 1 -maxdepth 1 -print -quit)" cp /source/auth.json /source/models.json /source/settings.json /target/agent/ chown -R 10001:10001 /target chmod 0700 /target /target/agent chmod 0600 /target/agent/auth.json /target/agent/models.json /target/agent/settings.json test "$(stat -c "%u:%g:%a" /target/agent/auth.json)" = 10001:10001:600 test "$(stat -c "%u:%g:%a" /target/agent/models.json)" = 10001:10001:600 test "$(stat -c "%u:%g:%a" /target/agent/settings.json)" = 10001:10001:600 ' } task13_prepare_local_application_secrets() { local owner_label owner_label="$(docker volume inspect --format '{{ index .Labels "io.thothii.task13.run" }}' \ "$TASK13_APPLICATION_SECRETS_VOLUME")" [[ "$owner_label" == "$TASK13_RUN_ID" ]] \ || task13_fail "application-secret runtime volume lacks the Task 13 run label" task13_run_logged "project local application secrets for the core runtime" docker run --rm \ --name "$TASK13_APPLICATION_SECRETS_PROJECTION_CONTAINER" \ --label "io.thothii.task13.run=$TASK13_RUN_ID" \ --user 0:0 \ --entrypoint sh \ --volume "$TASK13_SECRETS:/source/thothii.secrets:ro" \ --volume "$TASK13_SESSION_RUNTIME_PASSWORD:/source/task13-runtime-password:ro" \ --volume "$TASK13_SESSION_CA:/source/session_ca.pem:ro" \ --volume "$TASK13_APPLICATION_SECRETS_VOLUME:/target" \ "$TASK13_CORE_IMAGE" -ceu ' test -f /source/thothii.secrets && test ! -L /source/thothii.secrets test -f /source/task13-runtime-password && test ! -L /source/task13-runtime-password test -f /source/session_ca.pem && test ! -L /source/session_ca.pem test -z "$(find /target -mindepth 1 -maxdepth 1 -print -quit)" cp /source/thothii.secrets /source/task13-runtime-password /source/session_ca.pem /target/ cp /source/task13-runtime-password /target/session_runtime_password chown 0:0 /target chown 10001:10001 /target/thothii.secrets /target/task13-runtime-password /target/session_runtime_password /target/session_ca.pem chmod 0755 /target chmod 0600 /target/thothii.secrets /target/task13-runtime-password /target/session_runtime_password /target/session_ca.pem test "$(stat -c "%u:%g:%a" /target)" = 0:0:755 test "$(stat -c "%u:%g:%a" /target/thothii.secrets)" = 10001:10001:600 test "$(stat -c "%u:%g:%a" /target/task13-runtime-password)" = 10001:10001:600 test "$(stat -c "%u:%g:%a" /target/session_runtime_password)" = 10001:10001:600 test "$(stat -c "%u:%g:%a" /target/session_ca.pem)" = 10001:10001:600 ' } task13_prepare_registry_remote() { local owner_label owner_label="$(docker volume inspect --format '{{ index .Labels "io.thothii.task13.run" }}' \ "$TASK13_REGISTRY_RUNTIME_VOLUME")" [[ "$owner_label" == "$TASK13_RUN_ID" ]] \ || task13_fail "registry fixture runtime volume lacks the Task 13 run label" task13_run_logged "project Git fixture for the core runtime" docker run --rm \ --name "$TASK13_REGISTRY_PROJECTION_CONTAINER" \ --label "io.thothii.task13.run=$TASK13_RUN_ID" \ --user 0:0 \ --entrypoint sh \ --volume "$TASK13_REMOTE:/source:ro" \ --volume "$TASK13_REGISTRY_RUNTIME_VOLUME:/target" \ "$TASK13_CORE_IMAGE" -ceu ' test -f /source/HEAD && test -d /source/objects && test -d /source/refs cp -a /source/. /target/ chown -R 10001:10001 /target chmod -R u=rwX,go= /target test "$(stat -c "%u:%g:%a" /target)" = 10001:10001:700 test "$(stat -c "%u:%g" /target/HEAD)" = 10001:10001 ' } task13_start_stack() { printf '== Build and start isolated local Compose distribution ==\n' task13_assert_rendered_contract task13_compose_logged "build local Compose images" build --pull task13_compose_logged "create local core authentication runtime" create core task13_prepare_local_auth_runtime task13_prepare_local_pi_runtime task13_prepare_local_application_secrets task13_prepare_registry_remote task13_compose_start_logged "start local Compose distribution" up --detach --wait --wait-timeout 120 TASK13_NETWORK="$(docker network ls \ --filter "label=com.docker.compose.project=$TASK13_PROJECT" \ --filter 'label=com.docker.compose.network=thothii' --format '{{.Name}}')" [[ -n "$TASK13_NETWORK" && "$TASK13_NETWORK" != *$'\n'* ]] || task13_fail "isolated Compose network was not resolved" task13_run_logged "start deterministic local LLM fixture" docker run --detach \ --name "$TASK13_LLM_CONTAINER" \ --label "io.thothii.task13.run=$TASK13_RUN_ID" \ --network "$TASK13_NETWORK" \ --entrypoint node \ --volume "$TASK13_LLM_SERVER:/fixtures/fake-llm.mjs:ro" \ "$TASK13_CORE_IMAGE" /fixtures/fake-llm.mjs for _attempt in $(seq 1 30); do if docker exec "$TASK13_LLM_CONTAINER" node -e \ "fetch('http://127.0.0.1:9000/health',{signal:AbortSignal.timeout(3000)}).then(r=>{if(!r.ok)process.exit(1)}).catch(()=>process.exit(1))" \ >>"$TASK13_LOG" 2>&1; then return 0 fi sleep 1 done task13_log_failure "deterministic local LLM fixture readiness" } task13_start_server_stack() { printf '== Build and start isolated Linux server profile ==\n' task13_assert_rendered_contract task13_compose_logged "build server Compose images" build --pull core frontend task13_compose_logged "create server Compose resources" create core frontend TASK13_NETWORK="$(docker network ls \ --filter "label=com.docker.compose.project=$TASK13_PROJECT" \ --filter 'label=com.docker.compose.network=thothii' --format '{{.Name}}')" [[ -n "$TASK13_NETWORK" && "$TASK13_NETWORK" != *$'\n'* ]] \ || task13_fail "isolated server Compose network was not resolved" task13_run_logged "start scoped fake OIDC provider" docker run --detach \ --name "$TASK13_OIDC_CONTAINER" \ --label "io.thothii.task13.run=$TASK13_RUN_ID" \ --network "$TASK13_NETWORK" --network-alias task13-fake-oidc \ --user "$(id -u):$(id -g)" \ --env "TASK13_AUTHENTIK_API_TOKEN=$TASK13_AUTHENTIK_API_TOKEN" \ --env NODE_EXTRA_CA_CERTS=/fixtures/oidc-cert.pem \ --entrypoint node \ --volume "$TASK13_OIDC_SERVER:/fixtures/fake-oidc.mjs:ro" \ --volume "$TASK13_OIDC_CERT:/fixtures/oidc-cert.pem:ro" \ --volume "$TASK13_OIDC_KEY:/fixtures/oidc-key.pem:ro" \ "$TASK13_CORE_IMAGE" /fixtures/fake-oidc.mjs for _attempt in $(seq 1 30); do if docker exec "$TASK13_OIDC_CONTAINER" node -e \ "fetch('https://task13-fake-oidc:9443/health',{signal:AbortSignal.timeout(3000)}).then(r=>{if(!r.ok)process.exit(1)}).catch(()=>process.exit(1))" \ >>"$TASK13_LOG" 2>&1; then break fi sleep 1 done docker exec "$TASK13_OIDC_CONTAINER" node -e \ "fetch('https://task13-fake-oidc:9443/health',{signal:AbortSignal.timeout(3000)}).then(r=>{if(!r.ok)process.exit(1)}).catch(()=>process.exit(1))" \ >>"$TASK13_LOG" 2>&1 || task13_log_failure "scoped fake OIDC provider readiness" task13_compose_start_logged "start server Compose distribution" \ up --detach --wait --wait-timeout 120 core frontend } task13_frontend_address() { task13_compose port frontend 8080 | awk 'NR == 1 {print $0}' } task13_core_id() { task13_compose ps -q core } task13_assert_maintenance_auth_isolation() { local rendered rendered="$TASK13_TMP/maintenance-auth-isolation.json" if ! task13_compose --profile workspace-maintenance config --format json >"$rendered" 2>>"$TASK13_LOG"; then task13_log_failure "render workspace maintenance mount isolation" fi if ! node - "$rendered" <<'NODE' const config = JSON.parse(require("node:fs").readFileSync(process.argv[2], "utf8")); const maintenance = config.services?.["workspace-maintenance"]; if (!maintenance || !Array.isArray(maintenance.volumes)) process.exit(1); if (maintenance.volumes.some((mount) => mount?.target === "/run/thothii-auth" || mount?.target === "/data/auth")) { process.exit(1); } NODE then task13_log_failure "workspace maintenance authentication mount isolation" fi task13_compose_logged "seed core authentication isolation sentinel" exec -T core sh -ceu \ ': > /data/auth/task13-maintenance-isolation-sentinel' task13_compose_logged "workspace maintenance auth isolation" \ --profile workspace-maintenance run --rm --no-deps --entrypoint sh workspace-maintenance -ceu \ 'test ! -e /run/thothii-auth test -d /data/auth test ! -e /data/auth/task13-maintenance-isolation-sentinel' task13_compose_logged "verify core authentication isolation sentinel" exec -T core sh -ceu \ 'test -f /data/auth/task13-maintenance-isolation-sentinel' task13_compose_logged "remove core authentication isolation sentinel" exec -T core sh -ceu \ 'rm -f /data/auth/task13-maintenance-isolation-sentinel' } task13_assert_local_auth_lifecycle() { local frontend cookie_jar login_body me csrf unauthenticated frontend="$(task13_frontend_address)" cookie_jar="$TASK13_TMP/local-auth.cookies" login_body="$TASK13_TMP/local-auth-login.json" printf '{"username":"%s","password":"%s","remember":true}' \ "$TASK13_AUTH_ADMIN" "$TASK13_AUTH_PASSWORD" >"$login_body" chmod 0600 "$cookie_jar" "$login_body" 2>/dev/null || chmod 0600 "$login_body" task13_run_logged "local auth configuration" curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" \ --max-time "$TASK13_CURL_MAX_TIME" --fail --silent --show-error "http://$frontend/api/auth/config" task13_run_logged "local auth login" curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" \ --max-time "$TASK13_CURL_MAX_TIME" --fail --silent --show-error --cookie-jar "$cookie_jar" \ -H "Origin: http://$frontend" -H 'content-type: application/json' --data-binary "@$login_body" \ "http://$frontend/api/auth/local/login" me="$(curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" --max-time "$TASK13_CURL_MAX_TIME" \ --fail --silent --show-error --cookie "$cookie_jar" "http://$frontend/api/me")" node -e 'const value=JSON.parse(process.argv[1]); if(value.issuer!=="local"||value.session?.remembered!==true||typeof value.csrfToken!=="string") process.exit(1)' "$me" \ || task13_fail "local login did not create a remembered authenticated session" csrf="$(node -e 'process.stdout.write(JSON.parse(process.argv[1]).csrfToken)' "$me")" task13_compose_logged "remembered local auth core restart" up --detach --force-recreate --wait --wait-timeout 120 core me="$(curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" --max-time "$TASK13_CURL_MAX_TIME" \ --fail --silent --show-error --cookie "$cookie_jar" "http://$frontend/api/me")" node -e 'const value=JSON.parse(process.argv[1]); if(value.session?.remembered!==true) process.exit(1)' "$me" \ || task13_fail "remembered local session did not survive a core restart" csrf="$(node -e 'process.stdout.write(JSON.parse(process.argv[1]).csrfToken)' "$me")" task13_run_logged "local auth Pi management" curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" \ --max-time 45 --fail --silent --show-error --cookie "$cookie_jar" \ -H "Origin: http://$frontend" -H "x-thothii-csrf: $csrf" -X POST "http://$frontend/api/pi-management/test" task13_run_logged "local auth logout" curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" \ --max-time "$TASK13_CURL_MAX_TIME" --fail --silent --show-error --cookie "$cookie_jar" \ -H "Origin: http://$frontend" -H "x-thothii-csrf: $csrf" -X POST "http://$frontend/api/auth/logout" unauthenticated="$(curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" --max-time "$TASK13_CURL_MAX_TIME" \ --silent --output /dev/null --write-out '%{http_code}' --cookie "$cookie_jar" "http://$frontend/api/me")" [[ "$unauthenticated" == 401 ]] || task13_fail "local logout did not revoke the remembered session" } task13_create_admin_session() { local frontend login_body me frontend="$(task13_frontend_address)" TASK13_ADMIN_COOKIE="$TASK13_TMP/operations-admin.cookies" login_body="$TASK13_TMP/operations-admin-login.json" printf '{"username":"%s","password":"%s","remember":true}' \ "$TASK13_AUTH_ADMIN" "$TASK13_AUTH_PASSWORD" >"$login_body" chmod 0600 "$login_body" task13_run_logged "create authenticated smoke administration session" curl \ --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" --max-time "$TASK13_CURL_MAX_TIME" \ --fail --silent --show-error --cookie-jar "$TASK13_ADMIN_COOKIE" \ -H "Origin: http://$frontend" -H 'content-type: application/json' --data-binary "@$login_body" \ "http://$frontend/api/auth/local/login" me="$(curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" --max-time "$TASK13_CURL_MAX_TIME" \ --fail --silent --show-error --cookie "$TASK13_ADMIN_COOKIE" "http://$frontend/api/me")" TASK13_ADMIN_CSRF="$(node -e 'const value=JSON.parse(process.argv[1]); if(typeof value.csrfToken!=="string") process.exit(1); process.stdout.write(value.csrfToken)' "$me")" \ || task13_fail "authenticated smoke administration session lacks CSRF state" } task13_authenticated_get() { local path="$1" frontend frontend="$(task13_frontend_address)" curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" --max-time "$TASK13_CURL_MAX_TIME" \ --fail --silent --show-error --cookie "$TASK13_ADMIN_COOKIE" "http://$frontend/api/$path" } task13_authenticated_post() { local path="$1" frontend frontend="$(task13_frontend_address)" curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" --max-time 15 \ --fail --silent --show-error --cookie "$TASK13_ADMIN_COOKIE" \ -H "Origin: http://$frontend" -H "x-thothii-csrf: $TASK13_ADMIN_CSRF" \ -X POST "http://$frontend/api/$path" } task13_assert_local_restore_reauthentication() { local frontend archive login_body cookie_before cookie_after me status_before status_after frontend="$(task13_frontend_address)" archive="$TASK13_TMP/local-restore-source.zip" login_body="$TASK13_TMP/local-restore-login.json" cookie_before="$TASK13_TMP/local-restore-before.cookies" cookie_after="$TASK13_TMP/local-restore-after.cookies" printf '{"username":"%s","password":"%s","remember":true}' \ "$TASK13_AUTH_ADMIN" "$TASK13_AUTH_PASSWORD" >"$login_body" chmod 0600 "$login_body" task13_run_logged "create pre-backup browser session" curl \ --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" --max-time "$TASK13_CURL_MAX_TIME" \ --fail --silent --show-error --cookie-jar "$cookie_before" \ -H "Origin: http://$frontend" -H 'content-type: application/json' --data-binary "@$login_body" \ "http://$frontend/api/auth/local/login" task13_compose_logged "stop local stack for backup" stop task13_run_logged "create real default-custody backup" "$TASK13_THT" \ --installation "$TASK13_INSTALLATION" backup --output "$archive" python3 - "$archive" <<'PY' import json import sys import zipfile with zipfile.ZipFile(sys.argv[1]) as archive: manifest = json.loads(archive.read("manifest.json")) volumes = [item["logical_name"] for item in manifest["volumes"]] if volumes != ["embedding-models", "pi-state", "qdrant-data", "sessions", "settings", "workspace-registry", "workspace-secrets"]: raise SystemExit(f"unexpected backup volume custody: {volumes}") entries = manifest["entries"] if any(item.get("logical_name") == "auth-state" or "/data/auth" in item.get("source_path", "") for item in entries): raise SystemExit("default backup contains authentication runtime state") auth = [item for item in entries if item["path"].startswith("authentication-secrets/")] if len(auth) != 1 or not auth[0]["path"].endswith("-auth.yaml") or auth[0]["archived"]: raise SystemExit("default backup auth custody is not an auth.yaml reference only") if any(item["path"].endswith("users.yaml") for item in entries): raise SystemExit("default backup contains users.yaml") PY task13_compose_start_logged "restart local stack before restore" up --detach --wait --wait-timeout 120 status_before="$(curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" --max-time "$TASK13_CURL_MAX_TIME" \ --silent --output /dev/null --write-out '%{http_code}' --cookie "$cookie_before" "http://$frontend/api/me")" [[ "$status_before" == 200 ]] || task13_fail "pre-backup browser session did not survive an ordinary stop/start" task13_run_logged "create post-backup browser session" curl \ --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" --max-time "$TASK13_CURL_MAX_TIME" \ --fail --silent --show-error --cookie-jar "$cookie_after" \ -H "Origin: http://$frontend" -H 'content-type: application/json' --data-binary "@$login_body" \ "http://$frontend/api/auth/local/login" me="$(curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" --max-time "$TASK13_CURL_MAX_TIME" \ --fail --silent --show-error --cookie "$cookie_after" "http://$frontend/api/me")" node -e 'const value=JSON.parse(process.argv[1]); if(value.issuer!=="local"||value.session?.remembered!==true) process.exit(1)' "$me" \ || task13_fail "post-backup browser session was not authenticated" task13_compose_logged "seed valid pending OIDC state excluded from restore" exec -T core node --input-type=module -e ' const { loadConfig } = await import("/app/backend/dist/config.js"); const { createFileAuthSessionStore } = await import("/app/backend/dist/auth/session-store.js"); const config = loadConfig(process.env); const revision = config.authentication.current().revision; const store = createFileAuthSessionStore(config.authStateRoot); await store.createOidcState({ nonce: "n".repeat(43), codeVerifier: "v".repeat(43), returnTo: "/", authConfigRevision: revision, issuer: "https://pending.task13.invalid", browserTransactionDigest: "d".repeat(64), browserTransactionTransport: "loopback_http", }); ' task13_compose_logged "verify pre-restore authentication runtime population" exec -T core sh -ceu \ 'test "$(find /data/auth/sessions -type f | wc -l | tr -d " ")" -ge 2 && test -n "$(find /data/auth/oidc -type f -name "*.json" -print -quit)"' task13_run_logged "perform real running local production restore" "$TASK13_THT" \ --installation "$TASK13_INSTALLATION" restore "$archive" --yes status_before="$(curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" --max-time "$TASK13_CURL_MAX_TIME" \ --silent --output /dev/null --write-out '%{http_code}' --cookie "$cookie_before" "http://$frontend/api/me")" status_after="$(curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" --max-time "$TASK13_CURL_MAX_TIME" \ --silent --output /dev/null --write-out '%{http_code}' --cookie "$cookie_after" "http://$frontend/api/me")" [[ "$status_before" == 401 && "$status_after" == 401 ]] \ || task13_fail "restore did not force every independent browser session to reauthenticate" task13_compose_logged "verify private empty restored auth state" exec -T core sh -ceu ' test "$(stat -c %a /data/auth)" = 700 test "$(stat -c %a /data/auth/sessions)" = 700 test "$(stat -c %a /data/auth/oidc)" = 700 test "$(stat -c %u /data/auth)" = "$(id -u)" test -z "$(find /data/auth/sessions /data/auth/oidc -mindepth 1 -print -quit)" ' task13_create_admin_session } task13_assert_server_oidc_restore_verification() { local archive frontend status diagnostics restore_output restore_rc restore_cause local rollback_output rollback_rc rollback_sentinel provider_label checkpoint_leftover local registry_before registry_after integrity_output archive="$TASK13_TMP/server-oidc-restore-source.zip" frontend="$(task13_frontend_address)" task13_compose_logged "seed valid server authentication runtime excluded from restore" exec -T core node --input-type=module -e ' const { loadConfig } = await import("/app/backend/dist/config.js"); const { createFileAuthSessionStore } = await import("/app/backend/dist/auth/session-store.js"); const config = loadConfig(process.env); const revision = config.authentication.current().revision; const store = createFileAuthSessionStore(config.authStateRoot); await store.create({ principal: { issuer: "https://task13-fake-oidc:9443/application/o/task13/", subject: "restore-browser", roles: ["user"], permissions: ["session.use"], isAdmin: false }, method: "oidc", remembered: false, authConfigRevision: revision, idleTtlMs: 60_000, absoluteTtlMs: 120_000, }); await store.createOidcState({ nonce: "n".repeat(43), codeVerifier: "v".repeat(43), returnTo: "/", authConfigRevision: revision, issuer: "https://task13-fake-oidc:9443/application/o/task13/", browserTransactionDigest: "d".repeat(64), browserTransactionTransport: "https", }); ' task13_compose_logged "stop server stack for OIDC restore" stop rollback_sentinel="$TASK13_SERVER_DATA/task13-restore-rollback" printf 'backup-state\n' >"$rollback_sentinel" task13_run_logged "create real server default-custody backup" task13_server_tht \ backup --output "$archive" printf 'current-state\n' >"$rollback_sentinel" provider_label="$(docker container inspect --format '{{ index .Config.Labels "io.thothii.task13.run" }}' "$TASK13_OIDC_CONTAINER")" [[ "$provider_label" == "$TASK13_RUN_ID" ]] || task13_fail "fake OIDC provider ownership changed before rollback injection" task13_run_logged "inject post-mutation OIDC verification failure" docker stop "$TASK13_OIDC_CONTAINER" rollback_output="$TASK13_TMP/server-oidc-rollback.out" set +e task13_server_tht restore "$archive" --yes \ >"$rollback_output" 2>&1 rollback_rc=$? set -e [[ "$rollback_rc" -ne 0 ]] || task13_fail "server restore unexpectedly passed with its OIDC provider unavailable" grep -Eq 'restore verification doctor:|authentication diagnostics did not pass after restore' "$rollback_output" \ || task13_fail "server restore rollback injection did not reach post-mutation authentication verification" if grep -Fq "$TASK13_OIDC_CLIENT_SECRET" "$rollback_output" \ || grep -Fq "$TASK13_AUTHENTIK_API_TOKEN" "$rollback_output"; then task13_fail "failed server restore exposed fake-provider custody values" fi [[ "$(sudo -n -- cat -- "$rollback_sentinel")" == current-state ]] \ || task13_fail "failed server restore did not roll back the server data bind" checkpoint_leftover="$(task13_server_checkpoint_leftover)" [[ -z "$checkpoint_leftover" ]] || task13_fail "failed server restore retained its private recovery checkpoint" task13_compose_logged "verify failed restore cleared authentication runtime" \ run --rm --no-deps --no-TTY core sh -ceu ' test "$(stat -c %a /data/auth)" = 700 test "$(stat -c %a /data/auth/sessions)" = 700 test "$(stat -c %a /data/auth/oidc)" = 700 test "$(stat -c %u /data/auth)" = "$(id -u)" test -z "$(find /data/auth/sessions /data/auth/oidc -mindepth 1 -print -quit)" ' task13_run_logged "restore scoped fake OIDC provider after failure injection" docker start "$TASK13_OIDC_CONTAINER" for _attempt in $(seq 1 30); do if docker exec "$TASK13_OIDC_CONTAINER" node -e \ "fetch('https://task13-fake-oidc:9443/health',{signal:AbortSignal.timeout(3000)}).then(r=>{if(!r.ok)process.exit(1)}).catch(()=>process.exit(1))" \ >>"$TASK13_LOG" 2>&1; then break fi sleep 1 done docker exec "$TASK13_OIDC_CONTAINER" node -e \ "fetch('https://task13-fake-oidc:9443/health',{signal:AbortSignal.timeout(3000)}).then(r=>{if(!r.ok)process.exit(1)}).catch(()=>process.exit(1))" \ >>"$TASK13_LOG" 2>&1 || task13_log_failure "restored scoped fake OIDC provider readiness" printf 'Task 13 server rollback injection passed: exit=%s; recovery checkpoint removed.\n' "$rollback_rc" registry_before="$(task13_registry_filesystem_fingerprint "$TASK13_SERVER_REGISTRY")" integrity_output="$TASK13_TMP/server-workspace-integrity.json" if ! task13_compose run --rm --no-deps --no-TTY core \ node /app/backend/dist/operator-command.js workspace-integrity \ >"$integrity_output" 2>>"$TASK13_LOG"; then task13_log_failure "stopped read-only workspace registry verification" fi node -e ' const value=JSON.parse(require("fs").readFileSync(process.argv[1], "utf8")); const keys=Object.keys(value).sort().join(","); if(keys!=="fingerprint,ready,state,workspaces"||value.ready!==true||value.state!=="uninitialized"||value.workspaces!==0||!/^sha256:[0-9a-f]{64}$/.test(value.fingerprint)) process.exit(1); ' "$integrity_output" || task13_fail "stopped registry inspector returned malformed or additional output" registry_after="$(task13_registry_filesystem_fingerprint "$TASK13_SERVER_REGISTRY")" [[ "$registry_after" == "$registry_before" ]] \ || task13_fail "stopped restore verification mutated the workspace registry filesystem" restore_output="$TASK13_TMP/server-oidc-restore.out" set +e task13_server_tht restore "$archive" --yes \ >"$restore_output" 2>&1 restore_rc=$? set -e if [[ "$restore_rc" -ne 0 ]]; then restore_cause=restore-failed if grep -Fq 'restore verification workspace: validate restored workspace registry' "$restore_output"; then restore_cause=workspace-validator-rejected fi printf 'Task 13 stopped restore diagnostic: exit=%s cause=%s\n' \ "$restore_rc" "$restore_cause" >&2 task13_sanitize <"$restore_output" | tail -n 8 >&2 return "$restore_rc" fi checkpoint_leftover="$(task13_server_checkpoint_leftover)" [[ -z "$checkpoint_leftover" ]] || task13_fail "successful server restore retained its private recovery checkpoint" task13_compose_start_logged "start restored server stack" up --detach --wait --wait-timeout 120 core frontend frontend="$(task13_frontend_address)" status="" for _attempt in $(seq 1 30); do status="$(curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" --max-time "$TASK13_CURL_MAX_TIME" \ --silent --output /dev/null --write-out '%{http_code}' "http://$frontend/api/me" || true)" [[ "$status" == 401 ]] && break sleep 1 done [[ "$status" == 401 ]] \ || task13_fail "OIDC restore did not require browser reauthentication (HTTP ${status:-unavailable})" task13_compose_logged "verify private empty server authentication state" exec -T core sh -ceu ' test "$(stat -c %a /data/auth)" = 700 test "$(stat -c %a /data/auth/sessions)" = 700 test "$(stat -c %a /data/auth/oidc)" = 700 test "$(stat -c %u /data/auth)" = "$(id -u)" test -z "$(find /data/auth/sessions /data/auth/oidc -mindepth 1 -print -quit)" ' diagnostics="$TASK13_TMP/server-auth-diagnostics-after-restore.json" task13_server_tht auth check --json >"$diagnostics" node -e 'const value=JSON.parse(require("fs").readFileSync(process.argv[1], "utf8")); if(value.mode!=="oidc"||value.ready!==true||value.checks.length!==1||value.checks[0].code!=="auth_ready") process.exit(1)' "$diagnostics" \ || task13_fail "restored server did not retain strict fake-provider OIDC diagnostics" } task13_assert_runtime() { local frontend expected_pi actual_pi core_id frontend="$(task13_frontend_address)" expected_pi="$(sed -n 's/^ARG PI_VERSION=//p' "$TASK13_ROOT/docker/core.Dockerfile" | head -n 1)" actual_pi="$(task13_compose exec -T core pi --version | tr -d '\r\n')" [[ -n "$expected_pi" && "$actual_pi" == "$expected_pi" ]] || task13_fail "embedded Pi version mismatch" task13_run_logged "frontend health" curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" \ --max-time "$TASK13_CURL_MAX_TIME" --fail --silent --show-error "http://$frontend/" task13_run_logged "same-origin core health" curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" \ --max-time "$TASK13_CURL_MAX_TIME" --fail --silent --show-error "http://$frontend/api/health" task13_compose_logged "core non-root identity" exec -T core sh -ceu \ 'test "$(id -u)" = 10001' task13_compose_logged "embedded Pi executable" exec -T core sh -ceu \ 'command -v pi >/dev/null' task13_compose_logged "core Docker socket isolation" exec -T core sh -ceu \ 'test ! -e /var/run/docker.sock' task13_assert_local_auth_lifecycle task13_create_admin_session task13_authenticated_get workspace-registry/status >/dev/null \ || task13_fail "authenticated workspace registry bootstrap failed" task13_compose_logged "active workspace registry state" exec -T core sh -ceu \ 'test -f /data/workspace-registry/state/active.json' task13_compose_logged "mounted Pi auth readability" exec -T core sh -ceu \ 'test -r /home/thoth/.pi/agent/auth.json' task13_compose_logged "mounted application secret readability" exec -T core sh -ceu \ 'test -r /run/secrets/thothii.secrets' core_id="$(task13_core_id)" [[ "$(docker inspect --format '{{ index .Config.Labels "io.thothii.task13.run" }}' "$core_id")" == "$TASK13_RUN_ID" ]] \ || task13_fail "core lacks the explicit Task 13 resource label" task13_assert_maintenance_auth_isolation task13_run_logged "tht Pi doctor" "$TASK13_THT" --installation "$TASK13_INSTALLATION" pi doctor } task13_report_server_workspace_failure() { local status="$1" response="$2" printf 'authenticated server /api/workspaces returned HTTP %s\n' "$status" >&2 printf '%s\n' '--- sanitized server workspace response ---' >&2 if [[ -s "$response" ]]; then tail -c 16384 "$response" | task13_sanitize >&2 else printf '%s\n' '(empty response)' >&2 fi printf '%s\n' '--- sanitized registry integrity probe ---' >&2 { task13_compose exec -T core node --input-type=module -e ' const { loadConfig } = await import("/app/backend/dist/config.js"); const { WorkspaceRegistry } = await import("/app/backend/dist/workspaces/registry.js"); const registry = new WorkspaceRegistry(loadConfig(process.env).workspaceRegistry); try { const revisions = await registry.list(); for (const revision of revisions) await registry.read(revision.id); console.log(JSON.stringify({ ok: true, revisions: revisions.length })); } catch (error) { console.log(JSON.stringify({ ok: false, name: error instanceof Error ? error.name : "UnknownError", code: error && typeof error === "object" && "code" in error ? error.code : "unknown", message: error instanceof Error ? error.message : "Unknown registry failure", })); process.exitCode = 1; } ' 2>&1 || printf '%s\n' '(registry integrity probe unavailable)' } | tail -n 20 | task13_sanitize >&2 printf '%s\n' '--- sanitized core logs (last 100 lines) ---' >&2 { task13_compose logs --no-color --tail 100 core 2>&1 \ || printf '%s\n' '(core logs unavailable)' } | tail -n 100 | task13_sanitize >&2 } task13_assert_server_runtime() { local frontend unauthenticated trusted_header_status session_status diagnostics status provider_requests core_id frontend_id local expected_core_image expected_frontend_image frontend="$(task13_frontend_address)" task13_run_logged "server frontend health" curl \ --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" --max-time "$TASK13_CURL_MAX_TIME" \ --fail --silent --show-error "http://$frontend/" task13_run_logged "server same-origin core health" curl \ --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" --max-time "$TASK13_CURL_MAX_TIME" \ --fail --silent --show-error "http://$frontend/api/health" core_id="$(task13_core_id)" frontend_id="$(task13_compose ps -q frontend)" expected_core_image="$(docker image inspect --format '{{.Id}}' "$TASK13_CORE_IMAGE")" expected_frontend_image="$(docker image inspect --format '{{.Id}}' "$TASK13_FRONTEND_IMAGE")" [[ "$(docker inspect --format '{{.Image}}' "$core_id")" == "$expected_core_image" ]] \ || task13_fail "server core did not use the smoke-built core image" [[ "$(docker inspect --format '{{.Image}}' "$frontend_id")" == "$expected_frontend_image" ]] \ || task13_fail "server frontend did not use the smoke-built frontend image" task13_compose_logged "verify server runtime configuration and secret readability" exec -T core sh -ceu ' check_readable() { test -r "$1" || { printf "server precondition failed: unreadable %s\n" "$1" >&2; exit 1; }; } check_readable /run/thothii-auth/CURRENT projection_generation="$(node -e '\'' const fs = require("node:fs"); const selector = JSON.parse(fs.readFileSync("/run/thothii-auth/CURRENT", "utf8")); if (selector.version !== 1 || selector.state !== "ready" || typeof selector.generation !== "string" || !/^[0-9a-f]{64}$/.test(selector.generation)) { process.exit(1); } process.stdout.write(selector.generation); '\'')" || { printf "server precondition failed: invalid authentication projection selector\n" >&2; exit 1; } check_readable "/run/thothii-auth/generations/$projection_generation/auth.yaml" check_readable "/run/thothii-auth/generations/$projection_generation/manifest.json" test -d /data/auth || { printf "server precondition failed: missing /data/auth\n" >&2; exit 1; } test -z "${AUTH_MODE+x}" || { printf "server precondition failed: AUTH_MODE must be unset\n" >&2; exit 1; } test "$THT_SESSION_STORAGE" = postgres || { printf "server precondition failed: session storage\n" >&2; exit 1; } check_readable /run/secrets/thothii.secrets check_readable /run/secrets/session_runtime_password check_readable /run/secrets/session_ca.pem check_readable /app/harness/workspaces/server-sessions.yaml ' task13_mount_fingerprint | grep -Fq '/data = bind :' \ || task13_fail "server profile did not bind the disposable data root" task13_mount_fingerprint | grep -Fq '/home/thoth/.pi = bind :' \ || task13_fail "server profile did not bind the disposable Pi state root" task13_mount_fingerprint | grep -Fq '/data/workspace-registry = bind :' \ || task13_fail "server profile did not bind the disposable registry root" task13_assert_maintenance_auth_isolation unauthenticated="$(curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" \ --max-time "$TASK13_CURL_MAX_TIME" --silent --output /dev/null --write-out '%{http_code}' \ "http://$frontend/api/workspaces")" [[ "$unauthenticated" == 401 ]] || task13_fail "server profile did not enforce OIDC authentication" trusted_header_status="$(curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" \ --max-time "$TASK13_CURL_MAX_TIME" --silent --output /dev/null --write-out '%{http_code}' \ -H 'x-thoth-trusted-principal-issuer: task13-proxy' \ -H 'x-thoth-trusted-principal-subject: task13-user' \ -H 'x-thoth-trusted-principal-display-name: Task 13 User' \ -H 'x-thoth-trusted-is-admin: 0' \ "http://$frontend/api/workspaces")" [[ "$trusted_header_status" == 401 ]] || task13_fail "server accepted retired trusted identity headers" session_status="$(curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" \ --max-time "$TASK13_CURL_MAX_TIME" --silent --output "$TASK13_TMP/server-sessions.out" \ --write-out '%{http_code}' \ "http://$frontend/api/sessions")" [[ "$session_status" == 401 ]] \ || task13_fail "server session route did not fail closed before OIDC authentication" if grep -Fq "$TASK13_SECRET_VALUE" "$TASK13_TMP/server-sessions.out"; then task13_fail "server session failure exposed the fixture secret" fi status="$TASK13_TMP/server-auth-status.json" task13_run_logged "server static OIDC status" task13_server_tht auth status --json task13_server_tht auth status --json >"$status" node -e 'const value=JSON.parse(require("fs").readFileSync(process.argv[1], "utf8")); if(value.state!=="ready"||value.equal!==true||!/^[0-9a-f]{64}$/.test(value.generation)||value.canonicalRevision!==`sha256:${value.generation}`) process.exit(1)' "$status" \ || task13_fail "server authentication projection status was not valid" diagnostics="$TASK13_TMP/server-auth-diagnostics.json" task13_run_logged "server live OIDC diagnostics" task13_server_tht auth check --json task13_server_tht auth check --json >"$diagnostics" node -e 'const value=JSON.parse(require("fs").readFileSync(process.argv[1], "utf8")); if(value.mode!=="oidc"||value.ready!==true||value.checks.length!==1||value.checks[0].code!=="auth_ready") process.exit(1)' "$diagnostics" \ || task13_fail "scoped fake OIDC provider did not pass live production diagnostics" provider_requests="$(docker logs "$TASK13_OIDC_CONTAINER" 2>>"$TASK13_LOG")" [[ "$(grep -Fc 'group:task13-users' <<<"$provider_requests")" -ge 1 ]] \ || task13_fail "live OIDC diagnostics did not verify the mandatory user group" [[ "$(grep -Fc 'group:task13-admins' <<<"$provider_requests")" -ge 1 ]] \ || task13_fail "live OIDC diagnostics did not verify the mandatory administrator group" if grep -Fq 'group:task13-unrelated' <<<"$provider_requests" \ || grep -Fq 'task13-unrelated' "$diagnostics"; then task13_fail "live OIDC diagnostics queried or warned about an unrelated group" fi if grep -Fq "$TASK13_OIDC_CLIENT_SECRET" "$diagnostics" || grep -Fq "$TASK13_AUTHENTIK_API_TOKEN" "$diagnostics"; then task13_fail "OIDC diagnostics exposed a fixture secret" fi } task13_registry_status() { task13_authenticated_get workspace-registry/status } task13_registry_head() { sed -n 's/.*"head":"\([0-9a-f][0-9a-f]*\)".*/\1/p' } task13_active_registry_head() { task13_compose exec -T core sed -n \ 's/.*"head":"\([0-9a-f][0-9a-f]*\)".*/\1/p' \ /data/workspace-registry/state/active.json } task13_assert_sentinels() { task13_compose exec -T core sh -ceu ' test "$(cat /data/settings/task13-settings)" = settings-preserved test "$(cat /data/sessions/task13-session)" = sessions-preserved test "$(cat /home/thoth/.pi/task13-pi-state)" = pi-state-preserved test -f /data/workspace-registry/state/active.json ' } task13_mount_fingerprint() { docker inspect --format '{{range .Mounts}}{{println .Destination "=" .Type ":" .Name}}{{end}}' "$(task13_core_id)" \ | LC_ALL=C sort } task13_service_mount_fingerprint() { local service="$1" docker inspect --format '{{range .Mounts}}{{println .Destination "=" .Type ":" .Name}}{{end}}' \ "$(task13_compose ps -q "$service")" | LC_ALL=C sort } task13_prepare_persistence() { task13_compose exec -T core sh -ceu ' printf %s settings-preserved > /data/settings/task13-settings printf %s sessions-preserved > /data/sessions/task13-session printf %s pi-state-preserved > /home/thoth/.pi/task13-pi-state ' TASK13_INITIAL_MOUNTS="$(task13_mount_fingerprint)" TASK13_INITIAL_HEAD="$(task13_active_registry_head)" [[ "$TASK13_INITIAL_HEAD" =~ ^[0-9a-f]{40}$ ]] || task13_fail "initial registry head is invalid" task13_authenticated_get workspaces \ | grep -Fq 'Task 13 Smoke' || task13_fail "initial workspace is unavailable" } task13_registry_lifecycle() { local offline_status offline_head valid_head evidence_head repaired_head printf '== Recreate offline and retain the validated registry snapshot == ' task13_write_environment /fixtures/offline.git task13_compose_logged "offline Compose recreation" up --detach --force-recreate --wait --wait-timeout 120 offline_status="$(task13_registry_status)" offline_head="$(printf '%s' "$offline_status" | task13_registry_head)" [[ "$offline_head" == "$TASK13_INITIAL_HEAD" ]] || task13_fail "offline recreation changed registry head" grep -Fq '"degraded":true' <<<"$offline_status" || task13_fail "offline recreation did not report degraded mode" task13_assert_sentinels [[ "$(task13_mount_fingerprint)" == "$TASK13_INITIAL_MOUNTS" ]] || task13_fail "offline recreation changed volume identity" printf '== Pull a valid catalog+descriptor metadata update == ' task13_replace_once "$TASK13_SEED/$TASK13_WORKSPACE_ID/workspace.yaml" 'name: Task 13 Smoke' 'name: Task 13 Smoke Updated' task13_replace_once "$TASK13_SEED/thoth-workspaces.yaml" 'name: Task 13 Smoke' 'name: Task 13 Smoke Updated' task13_commit_registry_change 'Update Task 13 workspace metadata' task13_write_environment /fixtures/remote.git task13_compose_logged "online Compose recreation" up --detach --force-recreate --wait --wait-timeout 120 task13_authenticated_post workspace-registry/pull >/dev/null task13_authenticated_get workspaces | grep -Fq 'Task 13 Smoke Updated' || task13_fail "valid Git update was not activated" valid_head="$(task13_active_registry_head)" [[ "$valid_head" =~ ^[0-9a-f]{40}$ && "$valid_head" != "$TASK13_INITIAL_HEAD" ]] || task13_fail "valid Git update did not advance the registry head" TASK13_INITIAL_HEAD="$valid_head" task13_assert_sentinels printf '== Pull a content-only Git Evidence update == ' printf 'guide v2\n' >"$TASK13_SEED/$TASK13_WORKSPACE_ID/evidence/guide.md" task13_commit_registry_change 'Update Task 13 workspace evidence only' task13_authenticated_post workspace-registry/pull >/dev/null evidence_head="$(task13_active_registry_head)" [[ "$evidence_head" =~ ^[0-9a-f]{40}$ && "$evidence_head" != "$valid_head" ]] || task13_fail "content-only Git Evidence update did not advance the registry head" TASK13_INITIAL_HEAD="$evidence_head" task13_assert_sentinels printf '== Reject catalog/descriptor metadata mismatch and retain the valid snapshot == ' task13_replace_once "$TASK13_SEED/thoth-workspaces.yaml" 'name: Task 13 Smoke Updated' 'name: Task 13 Smoke Drift' task13_commit_registry_change 'Break Task 13 workspace metadata parity' if task13_authenticated_post workspace-registry/pull >>"$TASK13_LOG" 2>&1; then task13_fail "registry accepted catalog/descriptor metadata mismatch" fi [[ "$(task13_active_registry_head)" == "$TASK13_INITIAL_HEAD" ]] || task13_fail "metadata mismatch replaced the valid registry head" task13_authenticated_get workspaces | grep -Fq 'Task 13 Smoke Updated' || task13_fail "metadata mismatch displaced the valid workspace" task13_replace_once "$TASK13_SEED/thoth-workspaces.yaml" 'name: Task 13 Smoke Drift' 'name: Task 13 Smoke Updated' task13_commit_registry_change 'Restore Task 13 workspace metadata parity' task13_authenticated_post workspace-registry/pull >/dev/null repaired_head="$(task13_active_registry_head)" [[ "$repaired_head" =~ ^[0-9a-f]{40}$ && "$repaired_head" != "$TASK13_INITIAL_HEAD" ]] || task13_fail "metadata repair did not restore a fresh valid registry head" TASK13_INITIAL_HEAD="$repaired_head" printf '== Reject orphan descriptor directories not listed in the catalog == ' mkdir -p "$TASK13_SEED/orphan" cp "$TASK13_SEED/$TASK13_WORKSPACE_ID/workspace.yaml" "$TASK13_SEED/orphan/workspace.yaml" task13_replace_once "$TASK13_SEED/orphan/workspace.yaml" 'id: task13-smoke' 'id: orphan' task13_replace_once "$TASK13_SEED/orphan/workspace.yaml" 'name: Task 13 Smoke Updated' 'name: Orphan Workspace' task13_replace_once "$TASK13_SEED/orphan/workspace.yaml" 'collection: task13-smoke' 'collection: orphan' task13_replace_once "$TASK13_SEED/orphan/workspace.yaml" 'uri: task13-smoke/evidence' 'uri: orphan/evidence' mkdir -p "$TASK13_SEED/orphan/evidence" printf 'orphan guide\n' >"$TASK13_SEED/orphan/evidence/guide.md" task13_commit_registry_change 'Add orphan Task 13 workspace directory' if task13_authenticated_post workspace-registry/pull >>"$TASK13_LOG" 2>&1; then task13_fail "registry accepted orphan Task 13 descriptor directory" fi [[ "$(task13_active_registry_head)" == "$TASK13_INITIAL_HEAD" ]] || task13_fail "orphan descriptor directory replaced the valid registry head" task13_authenticated_get workspaces | grep -Fq 'Task 13 Smoke Updated' || task13_fail "orphan descriptor displaced the valid workspace" rm -rf "$TASK13_SEED/orphan" task13_commit_registry_change 'Remove orphan Task 13 workspace directory' task13_authenticated_post workspace-registry/pull >/dev/null TASK13_INITIAL_HEAD="$(task13_active_registry_head)" printf '== Reject the retired flat workspace layout and retain the valid snapshot == ' mkdir -p "$TASK13_SEED/workspaces" "$TASK13_SEED/workspace-content/$TASK13_WORKSPACE_ID/evidence" cp "$TASK13_ROOT/scripts/fixtures/workspace-registry-task13.yaml" "$TASK13_SEED/workspaces/$TASK13_WORKSPACE_ID.yaml" printf 'legacy guide\n' >"$TASK13_SEED/workspace-content/$TASK13_WORKSPACE_ID/evidence/guide.md" task13_commit_registry_change 'Reintroduce retired flat Task 13 workspace layout' if task13_authenticated_post workspace-registry/pull >>"$TASK13_LOG" 2>&1; then task13_fail "registry accepted the retired flat Task 13 workspace layout" fi [[ "$(task13_active_registry_head)" == "$TASK13_INITIAL_HEAD" ]] || task13_fail "retired flat workspace layout replaced the valid registry head" task13_authenticated_get workspaces | grep -Fq 'Task 13 Smoke Updated' || task13_fail "retired flat workspace layout displaced the valid workspace" task13_assert_sentinels } task13_prepare_bad_candidate() { task13_run_logged "pull pinned stopped-core candidate" docker image pull "$TASK13_BAD_CANDIDATE_IMAGE" TASK13_BAD_CANDIDATE_ID="$(docker image inspect --format '{{.Id}}' "$TASK13_BAD_CANDIDATE_IMAGE")" [[ "$TASK13_BAD_CANDIDATE_ID" =~ ^sha256:[0-9a-f]{64}$ ]] \ || task13_fail "bad candidate image identity was not resolved" task13_record_image_evidence "$TASK13_BAD_CANDIDATE_IMAGE" rollback-candidate task13_run_logged "prove bad candidate exits" docker run \ --name "$TASK13_BAD_CANDIDATE_CONTAINER" \ --label "io.thothii.task13.run=$TASK13_RUN_ID" \ "$TASK13_BAD_CANDIDATE_IMAGE" [[ "$(docker container inspect --format '{{.State.Running}}:{{.State.ExitCode}}' \ "$TASK13_BAD_CANDIDATE_CONTAINER")" == false:0 ]] \ || task13_fail "bad candidate did not reach the guaranteed stopped state" task13_remove_labeled_container "$TASK13_BAD_CANDIDATE_CONTAINER" } task13_update_rollback() { local before_image before_mounts before_head output rc phase after_image after_mounts after_head printf '== Inject a bad pinned Pi candidate and prove automatic rollback ==\n' task13_prepare_bad_candidate before_image="$(docker inspect --format '{{.Image}}' "$(task13_core_id)")" TASK13_PREVIOUS_IMAGE_ID="$before_image" before_mounts="$(task13_mount_fingerprint)" before_head="$(task13_active_registry_head)" output="$TASK13_TMP/tht-update.out" set +e "$TASK13_THT" --installation "$TASK13_INSTALLATION" pi update \ --version "$TASK13_BAD_PI_VERSION" --source pull --image "$TASK13_BAD_CANDIDATE_IMAGE" --yes \ >"$output" 2>&1 rc=$? set -e [[ "$rc" -ne 0 ]] || task13_fail "bad Pi candidate unexpectedly passed update verification" if grep -Fq "$TASK13_SECRET_VALUE" "$output"; then task13_fail "tht update output exposed the fixture secret" fi grep -Fq 'previous core image was restored' "$output" \ || { task13_sanitize <"$output" >&2; task13_fail "tht did not report automatic rollback"; } [[ -f "$TASK13_UPDATE_STATE" ]] || task13_fail "tht update state was not persisted" phase="$(sed -n 's/.*"phase": "\([^"]*\)".*/\1/p' "$TASK13_UPDATE_STATE" | head -n 1)" [[ "$phase" == rolled_back ]] || task13_fail "update state phase is not rolled_back" if grep -Fq "$TASK13_SECRET_VALUE" "$TASK13_UPDATE_STATE"; then task13_fail "update state exposed the fixture secret" fi task13_compose_files after_image="$(docker inspect --format '{{.Image}}' "$(task13_core_id)")" after_mounts="$(task13_mount_fingerprint)" after_head="$(task13_active_registry_head)" [[ "$after_image" == "$before_image" ]] || task13_fail "rollback did not restore the previous core image" [[ "$after_mounts" == "$before_mounts" ]] || task13_fail "rollback changed persistence volume identity" [[ "$after_head" == "$before_head" ]] || task13_fail "rollback changed the active registry revision" task13_assert_sentinels task13_run_logged "post-rollback tht doctor" \ "$TASK13_THT" --installation "$TASK13_INSTALLATION" pi doctor task13_authenticated_get workspaces \ | grep -Fq 'Task 13 Smoke' || task13_fail "rollback lost the active workspace" } task13_remove_labeled_container() { local name="$1" label [[ -n "$name" ]] || return 0 if ! docker container inspect "$name" >/dev/null 2>&1; then return 0 fi label="$(docker container inspect --format '{{ index .Config.Labels "io.thothii.task13.run" }}' "$name")" [[ "$label" == "$TASK13_RUN_ID" ]] || { printf 'refusing to remove foreign container %s\n' "$name" >&2 return 1 } task13_bounded "$TASK13_CLEANUP_TIMEOUT" "remove owned container" \ docker container rm --force "$name" >/dev/null } task13_remove_labeled_image() { local reference="$1" label [[ -n "$reference" ]] || return 0 if ! docker image inspect "$reference" >/dev/null 2>&1; then return 0 fi label="$(docker image inspect --format '{{ index .Config.Labels "io.thothii.task13.run" }}' "$reference")" [[ "$label" == "$TASK13_RUN_ID" ]] || { printf 'refusing to remove foreign image %s\n' "$reference" >&2 return 1 } task13_bounded "$TASK13_CLEANUP_TIMEOUT" "remove owned image" \ docker image rm "$reference" >/dev/null } task13_remove_transaction_image() { local reference="$1" expected_id="$2" actual_id [[ -n "$reference" ]] || return 0 if ! docker image inspect "$reference" >/dev/null 2>&1; then return 0 fi if [[ ! "$reference" =~ ^thothii-core:tht-[0-9a-f]{16}-(candidate|previous)$ \ || ! "$expected_id" =~ ^sha256:([0-9a-f]{64}|owned)$ ]]; then printf 'refusing to remove invalid transaction image reference %s\n' "$reference" >&2 return 1 fi actual_id="$(docker image inspect --format '{{.Id}}' "$reference")" [[ "$actual_id" == "$expected_id" ]] || { printf 'refusing to remove foreign transaction image %s\n' "$reference" >&2 return 1 } task13_bounded "$TASK13_CLEANUP_TIMEOUT" "remove owned transaction image" \ docker image rm "$reference" >/dev/null } task13_assert_project_ownership() { local kind id ids label for kind in container volume network; do if [[ "$kind" == container ]]; then if ! ids="$(docker container ls -aq --filter "label=com.docker.compose.project=$TASK13_PROJECT")"; then task13_fail "could not enumerate Compose project container resources" return 1 fi elif ! ids="$(docker "$kind" ls -q --filter "label=com.docker.compose.project=$TASK13_PROJECT")"; then task13_fail "could not enumerate Compose project $kind resources" return 1 fi while IFS= read -r id; do [[ -n "$id" ]] || continue case "$kind" in container) if ! label="$(docker container inspect --format '{{ index .Config.Labels "io.thothii.task13.run" }}' "$id")"; then task13_fail "could not inspect Compose project container resource" return 1 fi ;; volume) if ! label="$(docker volume inspect --format '{{ index .Labels "io.thothii.task13.run" }}' "$id")"; then task13_fail "could not inspect Compose project volume resource" return 1 fi ;; network) if ! label="$(docker network inspect --format '{{ index .Labels "io.thothii.task13.run" }}' "$id")"; then task13_fail "could not inspect Compose project network resource" return 1 fi ;; esac if [[ "$label" != "$TASK13_RUN_ID" ]]; then task13_fail "Compose project contains a foreign $kind resource" return 1 fi done <<<"$ids" done } task13_assert_built_image_ownership() { local image label for image in "$TASK13_CORE_IMAGE" "$TASK13_FRONTEND_IMAGE"; do label="$(docker image inspect --format '{{ index .Config.Labels "io.thothii.task13.run" }}' "$image")" [[ "$label" == "$TASK13_RUN_ID" ]] || task13_fail "built image lacks the Task 13 run label" done } task13_reclaim_server_fixture_ownership() { local host_uid host_gid [[ "${TASK13_PROFILE:-local}" == server ]] || return 0 [[ -n "${TASK13_TMP:-}" && -d "$TASK13_TMP" ]] || return 0 [[ "${TASK13_TMP%/*}" == "${TASK13_TMP_PARENT:-}" \ && "${TASK13_TMP##*/}" == thothii-task13.* ]] \ || task13_fail "refusing to reclaim an unexpected server fixture path" [[ -n "${TASK13_CONTROL_DIR:-}" \ && "$TASK13_CONTROL_DIR" == "$TASK13_ROOT/.tht/$TASK13_PROJECT" \ && "$TASK13_PROJECT" =~ ^thothii-[0-9a-f]{12}$ ]] \ || task13_fail "refusing to reclaim an unexpected server control path" host_uid="$(id -u)" host_gid="$(id -g)" task13_bounded "$TASK13_CLEANUP_TIMEOUT" "reclaim server fixture ownership" \ sudo -n -- chown -hR "$host_uid:$host_gid" "$TASK13_TMP" "$TASK13_CONTROL_DIR" } task13_cleanup() { local original_rc="$1" cleanup_rc=0 transaction="" leftovers="" image_id="" set +e if [[ "$original_rc" -ne 0 && -n "${TASK13_PROJECT:-}" \ && -s "${TASK13_IMAGE_EVIDENCE_RECORDS:-}" && -n "${TASK13_IMAGE_EVIDENCE_OUTPUT:-}" ]]; then if ! task13_capture_failed_image_evidence >>"${TASK13_LOG:-/dev/null}" 2>&1; then rm -f "$TASK13_IMAGE_EVIDENCE_OUTPUT" fi fi if [[ "$original_rc" -ne 0 && "${TASK13_FAILURE_LOGGED:-0}" -eq 0 ]] \ && [[ -n "${TASK13_LOG:-}" && -f "$TASK13_LOG" ]]; then printf '%s\n' '--- sanitized Task 13 diagnostic log ---' >&2 tail -n 200 "$TASK13_LOG" | task13_sanitize >&2 fi task13_remove_labeled_container "${TASK13_BAD_CANDIDATE_CONTAINER:-}" || cleanup_rc=1 task13_remove_labeled_container "${TASK13_AUTH_PROJECTION_CONTAINER:-}" || cleanup_rc=1 task13_remove_labeled_container "${TASK13_PI_PROJECTION_CONTAINER:-}" || cleanup_rc=1 task13_remove_labeled_container "${TASK13_REGISTRY_PROJECTION_CONTAINER:-}" || cleanup_rc=1 task13_remove_labeled_container "${TASK13_LLM_CONTAINER:-}" || cleanup_rc=1 task13_remove_labeled_container "${TASK13_OIDC_CONTAINER:-}" || cleanup_rc=1 if [[ -n "${TASK13_PROJECT:-}" && -n "${TASK13_ROOT:-}" && -f "${TASK13_OVERRIDE:-}" ]]; then if task13_assert_project_ownership >>"${TASK13_LOG:-/dev/null}" 2>&1; then task13_compose_files if ! task13_bounded "$TASK13_CLEANUP_TIMEOUT" "stop owned Compose project" \ "${TASK13_COMPOSE[@]}" down --volumes --remove-orphans --timeout 10 \ >>"${TASK13_LOG:-/dev/null}" 2>&1; then cleanup_rc=1 fi else cleanup_rc=1 fi fi if ! { task13_reclaim_server_fixture_ownership } >>"${TASK13_LOG:-/dev/null}" 2>&1; then cleanup_rc=1 fi if [[ -f "${TASK13_UPDATE_STATE:-}" ]]; then transaction="$(sed -n 's/.*"transaction": "\([^"]*\)".*/\1/p' "$TASK13_UPDATE_STATE" | head -n 1)" fi if [[ -n "$transaction" ]]; then task13_remove_transaction_image "thothii-core:tht-$transaction-candidate" \ "${TASK13_BAD_CANDIDATE_ID:-}" >>"${TASK13_LOG:-/dev/null}" 2>&1 || cleanup_rc=1 task13_remove_transaction_image "thothii-core:tht-$transaction-previous" \ "${TASK13_PREVIOUS_IMAGE_ID:-}" >>"${TASK13_LOG:-/dev/null}" 2>&1 || cleanup_rc=1 fi for image in \ "${TASK13_FRONTEND_IMAGE:-}" \ "${TASK13_CORE_IMAGE:-}"; do [[ -n "$image" ]] || continue task13_remove_labeled_image "$image" >>"${TASK13_LOG:-/dev/null}" 2>&1 || cleanup_rc=1 done if [[ -n "${TASK13_RUN_ID:-}" ]]; then while IFS= read -r image_id; do [[ -n "$image_id" ]] || continue task13_remove_labeled_image "$image_id" >>"${TASK13_LOG:-/dev/null}" 2>&1 || cleanup_rc=1 done < <(docker image ls -q --filter "label=io.thothii.task13.run=$TASK13_RUN_ID" | sort -u) fi if [[ -n "${TASK13_CONTROL_DIR:-}" ]]; then if [[ "$TASK13_CONTROL_DIR" == "$TASK13_ROOT/.tht/$TASK13_PROJECT" \ && "$TASK13_PROJECT" =~ ^thothii-[0-9a-f]{12}$ ]]; then rm -rf "$TASK13_CONTROL_DIR" else cleanup_rc=1 fi fi if [[ -n "${TASK13_RUN_ID:-}" ]]; then leftovers="$(docker container ls -aq --filter "label=io.thothii.task13.run=$TASK13_RUN_ID")" leftovers+="$(docker volume ls -q --filter "label=io.thothii.task13.run=$TASK13_RUN_ID")" leftovers+="$(docker network ls -q --filter "label=io.thothii.task13.run=$TASK13_RUN_ID")" leftovers+="$(docker image ls -q --filter "label=io.thothii.task13.run=$TASK13_RUN_ID")" [[ -z "$leftovers" ]] || cleanup_rc=1 fi if [[ -n "${TASK13_TMP:-}" && -d "$TASK13_TMP" ]]; then if [[ "${TASK13_TMP%/*}" == "${TASK13_TMP_PARENT:-}" \ && "${TASK13_TMP##*/}" == thothii-task13.* ]]; then rm -rf "$TASK13_TMP" else cleanup_rc=1 fi fi if [[ "$cleanup_rc" -eq 0 ]]; then printf 'Task 13 cleanup proof: no labeled containers, volumes, networks, or images remain for %s.\n' \ "${TASK13_RUN_ID:-unknown}" else printf 'Task 13 cleanup proof failed for %s.\n' "${TASK13_RUN_ID:-unknown}" >&2 fi trap - EXIT if [[ "$original_rc" -ne 0 ]]; then exit "$original_rc" fi exit "$cleanup_rc" } task13_self_test_sanitizer() { local input output leaked TASK13_SECRET_VALUE="fixture-known-secret" input="$(printf '%s\n' \ 'fixture-known-secret' \ 'password=plain-secret' \ '{"api_key":"json-secret"}' \ 'Authorization: Bearer bearer-secret' \ 'https://alice:url-secret@example.invalid/repo.git')" output="$(printf '%s\n' "$input" | task13_sanitize)" for leaked in fixture-known-secret plain-secret json-secret bearer-secret url-secret; do if grep -Fq "$leaked" <<<"$output"; then task13_fail "sanitizer leaked $leaked" fi done [[ "$(grep -Fc '[REDACTED]' <<<"$output")" -eq 5 ]] \ || task13_fail "sanitizer did not redact every credential form" } task13_self_test_server_workspace_diagnostics() { local response output count i=1 response="$(mktemp "${TMPDIR:-/tmp}/thothii-task13-server-response.XXXXXX")" TASK13_SECRET_VALUE="fixture-known-secret" printf '%s\n' \ '{"error":"workspace_invalid","detail":"password=fixture-known-secret"}' >"$response" task13_compose() { if [[ "$*" == "exec -T core node --input-type=module -e "* ]]; then printf '%s\n' \ '{"name":"WorkspaceRegistryError","code":"workspace_invalid","message":"Workspace snapshot integrity check failed"}' return 0 fi [[ "$*" == "logs --no-color --tail 100 core" ]] \ || task13_fail "server diagnostics requested an unexpected Compose command" while [[ "$i" -le 150 ]]; do printf 'core-log-%03d token=fixture-known-secret\n' "$i" i=$((i + 1)) done } if ! output="$(task13_report_server_workspace_failure 400 "$response" 2>&1)"; then unset -f task13_compose rm -f "$response" task13_fail "server workspace diagnostics could not be captured" fi unset -f task13_compose rm -f "$response" [[ "$output" == *'authenticated server /api/workspaces returned HTTP 400'* ]] \ || task13_fail "server diagnostics omit the unexpected HTTP status" [[ "$output" == *'workspace_invalid'* ]] \ || task13_fail "server diagnostics omit the generic response" [[ "$output" == *'Workspace snapshot integrity check failed'* ]] \ || task13_fail "server diagnostics omit the bounded internal registry reason" [[ "$output" == *'core-log-051'* && "$output" != *'core-log-050'* ]] \ || task13_fail "server diagnostics do not bound core logs to the last 100 lines" count="$(grep -Ec '^core-log-[0-9]{3}' <<<"$output")" [[ "$count" -eq 100 ]] || task13_fail "server diagnostics emitted $count core log lines" [[ "$output" != *'fixture-known-secret'* ]] \ || task13_fail "server diagnostics leaked the fixture secret" [[ "$(grep -Fc '[REDACTED]' <<<"$output")" -ge 101 ]] \ || task13_fail "server diagnostics did not sanitize response and core logs" } task13_self_test_core_startup_diagnostics() { local output TASK13_SECRET_VALUE="fixture-known-secret" task13_compose() { case "$*" in "ps --all -q core") printf '%s\n' 'task13-core-id' ;; "logs --no-color --tail 100 core") printf '%s\n' \ 'Error: EACCES: permission denied, mkdir /data/auth/sessions' \ 'password=plain-secret token=fixture-known-secret' \ 'secret path: /run/secrets/private-token' \ ' at createFileAuthSessionStore (/app/backend/dist/auth/session-store.js:101:9)' ;; *) task13_fail "startup diagnostics requested an unexpected Compose command: $*" ;; esac } docker() { [[ "$*" == "inspect --format {{.State.Status}}:{{.State.ExitCode}} task13-core-id" ]] \ || task13_fail "startup diagnostics requested an unexpected Docker command: $*" printf '%s\n' 'exited:1' } output="$(task13_report_core_startup_failure 2>&1)" unset -f task13_compose docker [[ "$output" == 'Core startup cause: authentication state storage is unavailable (exit code 1).' ]] \ || task13_fail "startup diagnostics emitted a non-allowlisted cause: $output" for leaked in EACCES permission /data/auth /run/secrets plain-secret fixture-known-secret \ createFileAuthSessionStore session-store.js; do [[ "$output" != *"$leaked"* ]] || task13_fail "startup diagnostics leaked $leaked" done } task13_self_test_cleanup_ownership() { local calls foreign_error owned_name foreign_name calls="$(mktemp "${TMPDIR:-/tmp}/thothii-task13-cleanup-contract.XXXXXX")" foreign_error="$calls.foreign-error" owned_name="task13-owned-contract" foreign_name="task13-foreign-contract" TASK13_RUN_ID="task13-contract-run" docker() { printf '%s\n' "$*" >>"$calls" if [[ "$1 $2" == "container inspect" ]]; then if [[ "$3" == "--format" ]]; then if [[ "${*: -1}" == "$owned_name" ]]; then printf '%s\n' "$TASK13_RUN_ID" else printf '%s\n' 'some-other-run' fi fi return 0 fi [[ "$1 $2" == "container rm" ]] } if task13_remove_labeled_container "$foreign_name" 2>"$foreign_error"; then unset -f docker rm -f "$calls" "$foreign_error" task13_fail "cleanup accepted a foreign-labeled container" fi if grep -Fq "container rm --force $foreign_name" "$calls"; then unset -f docker rm -f "$calls" "$foreign_error" task13_fail "cleanup attempted to remove a foreign-labeled container" fi grep -Fq "refusing to remove foreign container $foreign_name" "$foreign_error" \ || task13_fail "cleanup refusal was not explicit" task13_remove_labeled_container "$owned_name" [[ "$(grep -Fc "container rm --force $owned_name" "$calls")" -eq 1 ]] \ || task13_fail "cleanup did not remove exactly the owned container" unset -f docker rm -f "$calls" "$foreign_error" } task13_self_test_image_cleanup_ownership() { local calls foreign_error owned_ref foreign_ref calls="$(mktemp "${TMPDIR:-/tmp}/thothii-task13-image-cleanup-contract.XXXXXX")" foreign_error="$calls.foreign-error" owned_ref="task13-owned-contract:local" foreign_ref="task13-foreign-contract:local" TASK13_RUN_ID="task13-contract-run" if ! declare -F task13_remove_labeled_image >/dev/null; then rm -f "$calls" "$foreign_error" task13_fail "image cleanup ownership guard is missing" fi docker() { printf '%s\n' "$*" >>"$calls" if [[ "$1 $2" == "image inspect" ]]; then if [[ "$3" == "--format" ]]; then if [[ "${*: -1}" == "$owned_ref" ]]; then printf '%s\n' "$TASK13_RUN_ID" else printf '%s\n' 'some-other-run' fi fi return 0 fi [[ "$1 $2" == "image rm" ]] } if task13_remove_labeled_image "$foreign_ref" 2>"$foreign_error"; then unset -f docker rm -f "$calls" "$foreign_error" task13_fail "cleanup accepted a foreign-labeled image" fi if grep -Fq "image rm $foreign_ref" "$calls"; then unset -f docker rm -f "$calls" "$foreign_error" task13_fail "cleanup attempted to remove a foreign-labeled image" fi grep -Fq "refusing to remove foreign image $foreign_ref" "$foreign_error" \ || task13_fail "image cleanup refusal was not explicit" task13_remove_labeled_image "$owned_ref" [[ "$(grep -Fc "image rm $owned_ref" "$calls")" -eq 1 ]] \ || task13_fail "cleanup did not remove exactly the owned image reference" unset -f docker rm -f "$calls" "$foreign_error" } task13_self_test_transaction_image_cleanup() { local calls foreign_error owned_ref foreign_ref calls="$(mktemp "${TMPDIR:-/tmp}/thothii-task13-transaction-cleanup-contract.XXXXXX")" foreign_error="$calls.foreign-error" owned_ref="thothii-core:tht-0123456789abcdef-candidate" foreign_ref="thothii-core:tht-fedcba9876543210-candidate" if ! declare -F task13_remove_transaction_image >/dev/null; then rm -f "$calls" "$foreign_error" task13_fail "transaction image cleanup guard is missing" fi docker() { printf '%s\n' "$*" >>"$calls" if [[ "$1 $2" == "image inspect" ]]; then if [[ "$3" == "--format" ]]; then if [[ "${*: -1}" == "$owned_ref" ]]; then printf '%s\n' 'sha256:owned' else printf '%s\n' 'sha256:foreign' fi fi return 0 fi [[ "$1 $2" == "image rm" ]] } if task13_remove_transaction_image "$foreign_ref" 'sha256:owned' 2>"$foreign_error"; then unset -f docker rm -f "$calls" "$foreign_error" task13_fail "cleanup accepted a transaction image with a foreign identity" fi if grep -Fq "image rm $foreign_ref" "$calls"; then unset -f docker rm -f "$calls" "$foreign_error" task13_fail "cleanup attempted to remove a foreign transaction image" fi grep -Fq "refusing to remove foreign transaction image $foreign_ref" "$foreign_error" \ || task13_fail "transaction image cleanup refusal was not explicit" task13_remove_transaction_image "$owned_ref" 'sha256:owned' [[ "$(grep -Fc "image rm $owned_ref" "$calls")" -eq 1 ]] \ || task13_fail "cleanup did not remove exactly the owned transaction image reference" unset -f docker rm -f "$calls" "$foreign_error" } task13_self_test_image_evidence() ( local fixture records output fixture="$(mktemp -d "${TMPDIR:-/tmp}/thothii-task15-image-evidence.XXXXXX")" records="$fixture/records.tsv" output="$fixture/images.json" trap 'rm -rf "$fixture"' EXIT : >"$records" TASK13_RUN_ID="task15-image-run" TASK13_SOURCE_COMMIT="0123456789abcdef0123456789abcdef01234567" TASK13_IMAGE_EVIDENCE_STATUS="fail" TASK13_IMAGE_EVIDENCE_RECORDS="$records" TASK13_IMAGE_EVIDENCE_OUTPUT="$output" TASK13_PROJECT="task15-image-project" docker() { case "$*" in "container ls -aq --filter label=com.docker.compose.project=task15-image-project") printf '%s\n' container-one container-two ;; "container inspect --format {{.Image}} container-one") printf '%s\n' 'sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' ;; "container inspect --format {{.Image}} container-two") printf '%s\n' 'sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb' ;; "image inspect --format {{.Id}} fixture-candidate@sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc") printf '%s\n' 'sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc' ;; "image inspect --format {{.Id}} sha256:"*) printf '%s\n' "${*: -1}" ;; "image inspect --format {{json .RepoDigests}} sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") printf '%s\n' '["fixture/core@sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"]' ;; "image inspect --format {{json .RepoDigests}} sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb") printf '%s\n' '[]' ;; "image inspect --format {{json .RepoDigests}} sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc") printf '%s\n' '["fixture/candidate@sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"]' ;; *) task13_fail "unexpected image evidence Docker command" ;; esac } task13_record_image_evidence \ 'fixture-candidate@sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc' \ 'rollback-candidate' task13_capture_failed_image_evidence unset -f docker node - "$output" <<'NODE' const manifest = JSON.parse(require("node:fs").readFileSync(process.argv[2], "utf8")); if (manifest.source_commit !== "0123456789abcdef0123456789abcdef01234567" || manifest.run_id !== "task15-image-run" || manifest.status !== "fail" || manifest.images.length !== 3) process.exit(1); const ids = manifest.images.map((image) => image.id); if (new Set(ids).size !== 3 || ids.some((id) => !/^sha256:[0-9a-f]{64}$/.test(id))) process.exit(1); if (!manifest.images.some((image) => image.roles.includes("rollback-candidate") && image.repo_digests[0] === "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc")) process.exit(1); NODE ) task13_self_test_rollback_fixture_contract() { [[ "${TASK13_BAD_CANDIDATE_BEHAVIOR:-}" == stopped ]] \ || task13_fail "rollback candidate is not declared as guaranteed stopped" [[ "$TASK13_BAD_CANDIDATE_IMAGE" =~ ^hello-world@sha256:[0-9a-f]{64}$ ]] \ || task13_fail "rollback candidate is not the digest-pinned stopped fixture" } task13_self_test_runtime_binding_fixture() { local root root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)" "$root/scripts/test-task13-runtime-fixtures.sh" local } task13_self_test_server_runtime_binding_fixture() { local root root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)" "$root/scripts/test-task13-runtime-fixtures.sh" server } task13_self_test_stopped_project_containers() { local calls foreign_error calls="$(mktemp "${TMPDIR:-/tmp}/thothii-task13-project-containers.XXXXXX")" foreign_error="$calls.foreign-error" TASK13_PROJECT="thothii-0123456789ab" TASK13_RUN_ID="task13-contract-run" docker() { printf '%s\n' "$*" >>"$calls" case "$1 $2 $3" in "container ls -aq") printf '%s\n' stopped-foreign ;; "container inspect --format") printf '%s\n' some-other-run ;; "volume ls -q"|"network ls -q") : ;; *) return 1 ;; esac } if task13_assert_project_ownership 2>"$foreign_error"; then unset -f docker rm -f "$calls" "$foreign_error" task13_fail "project cleanup accepted a stopped foreign container" fi grep -Fq 'container ls -aq' "$calls" \ || task13_fail "project cleanup did not enumerate stopped containers" grep -Fq 'Compose project contains a foreign container resource' "$foreign_error" \ || task13_fail "stopped foreign container refusal was not explicit" : >"$calls" docker() { printf '%s\n' "$*" >>"$calls" case "$1 $2 $3" in "container ls -aq") printf '%s\n' stopped-owned ;; "container inspect --format") printf '%s\n' "$TASK13_RUN_ID" ;; "volume ls -q"|"network ls -q") : ;; *) return 1 ;; esac } task13_assert_project_ownership [[ "$(grep -Fc 'container inspect --format' "$calls")" -eq 1 ]] \ || task13_fail "project cleanup did not inspect exactly the stopped owned container" unset -f docker rm -f "$calls" "$foreign_error" } task13_self_test_timeout_process_group() { local child_file child_pid="" rc child_file="$(mktemp "${TMPDIR:-/tmp}/thothii-task13-timeout-child.XXXXXX")" set +e task13_bounded 1 "child-process regression" bash -c \ 'sleep 30 & printf "%s\n" "$!" >"$1"; wait' _ "$child_file" >/dev/null 2>&1 rc=$? set -e child_pid="$(sed -n '1p' "$child_file")" [[ "$rc" -ne 0 ]] || { rm -f "$child_file" task13_fail "timed command unexpectedly succeeded" } if [[ -n "$child_pid" ]] && kill -0 "$child_pid" 2>/dev/null; then kill -KILL "$child_pid" 2>/dev/null || true rm -f "$child_file" task13_fail "timed command left its child process alive" fi rm -f "$child_file" } task13_self_test_nested_timeout_process_group() { local child_file child_pid="" rc child_file="$(mktemp "${TMPDIR:-/tmp}/thothii-task13-nested-timeout.XXXXXX")" task13_nested_timeout_fixture() { task13_bounded 30 "nested child-process regression" bash -c \ 'sleep 30 & printf "%s\n" "$!" >"$1"; wait' _ "$child_file" } set +e task13_supervise 1 "nested timeout supervisor" task13_nested_timeout_fixture >/dev/null 2>&1 rc=$? set -e unset -f task13_nested_timeout_fixture child_pid="$(sed -n '1p' "$child_file")" [[ "$rc" -ne 0 ]] || { rm -f "$child_file" task13_fail "nested timed command unexpectedly succeeded" } if [[ -n "$child_pid" ]] && kill -0 "$child_pid" 2>/dev/null; then kill -KILL "$child_pid" 2>/dev/null || true rm -f "$child_file" task13_fail "outer timeout left its nested command child alive" fi rm -f "$child_file" } task13_self_test_public_timeout_contract() { local root root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)" grep -Eq 'task13_supervise[[:space:]].*task13_smoke_main[[:space:]]+full' \ "$root/scripts/unified-deployment-smoke.sh" \ || task13_fail "direct unified smoke invocation lacks an internal supervisor" grep -Eq 'task13_supervise[[:space:]].*task13_smoke_main[[:space:]]+update' \ "$root/scripts/tht-update-smoke.sh" \ || task13_fail "direct update smoke invocation lacks an internal supervisor" grep -Eq 'task13_supervise[[:space:]].*task13_internal_semantic_smoke_main' \ "$root/scripts/internal-semantic-smoke.sh" \ || task13_fail "direct internal semantic smoke invocation lacks an internal supervisor" } task13_self_test_internal_semantic_offline_contract() { local root script root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)" script="$root/scripts/internal-semantic-smoke.sh" grep -Fq 'task13_write_offline_semantic_override' "$script" \ || task13_fail "internal semantic smoke lacks a dedicated offline override" grep -Fq 'task13_offline_semantic_compose_logged "offline semantic recreation" \' "$script" \ || task13_fail "offline semantic recreation must use the isolated offline compose wrapper" grep -Fq 'up --detach --wait --wait-timeout 120 --pull never qdrant embedding' "$script" \ || task13_fail "offline semantic recreation must start only qdrant and embedding with --pull never" grep -Eq 'down --remove-orphans --timeout 10$' "$script" \ || task13_fail "offline semantic phase must stop the stack before isolated recreation" grep -Fq 'internal: true' "$script" \ || task13_fail "offline semantic override must disable network egress" if grep -Eq 'offline semantic recreation.*embedding-model-init|offline semantic recreation.*core|offline semantic recreation.*frontend' "$script"; then task13_fail "offline semantic recreation must exclude bootstrap and non-semantic services" fi } task13_self_test_windows_release_contract() { local root script workflow root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)" script="$root/scripts/test-windows-clone-contract.ps1" workflow="$root/.github/workflows/deployment.yml" grep -Fq 'Task 13 path with spaces' "$script" \ || task13_fail "Windows contract does not operate from a path containing spaces" grep -Fq 'DockerStartup' "$script" \ || task13_fail "Windows contract lacks an explicit Docker startup mode" grep -Fq 'Kill($true)' "$script" \ || task13_fail "Windows bounded runner does not kill the full process tree" grep -Fq 'docker-desktop' "$workflow" \ || task13_fail "workflow lacks a manual self-hosted Windows Docker Desktop gate" grep -Fq -- '-DockerStartup' "$workflow" \ || task13_fail "manual Windows release job does not execute Docker startup mode" } task13_self_test_server_release_contract() { local root workflow server_smoke root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)" workflow="$root/.github/workflows/deployment.yml" server_smoke="$root/scripts/server-deployment-smoke.sh" [[ -x "$server_smoke" ]] || task13_fail "bounded Linux server deployment smoke is missing" grep -Fq 'deploy/compose.server.yaml' "$root/scripts/unified-deployment-smoke.sh" \ || task13_fail "server smoke does not load the server profile" grep -Fq 'deploy/compose.session-server.yaml.example' "$root/scripts/unified-deployment-smoke.sh" \ || task13_fail "server smoke does not load the required session overlay" grep -Eq 'timeout .*scripts/server-deployment-smoke\.sh' "$workflow" \ || task13_fail "workflow lacks an outer timeout for the Linux server smoke" } task13_self_test_registry_fingerprint() { local fixture fixture_escaped before after linked fixture="$(mktemp -d "${TMPDIR:-/tmp}/thothii-task13-registry-fingerprint.XXXXXX")" printf -v fixture_escaped '%q' "$fixture" trap "rm -rf -- $fixture_escaped; trap - RETURN" RETURN before="$(task13_registry_filesystem_fingerprint "$fixture")" [[ "$before" =~ ^sha256:[0-9a-f]{64}$ ]] \ || task13_fail "registry fingerprint did not return a bounded digest" [[ "$(task13_registry_filesystem_fingerprint "$fixture")" == "$before" ]] \ || task13_fail "registry fingerprint changed without a filesystem mutation" mkdir "$fixture/locks" after="$(task13_registry_filesystem_fingerprint "$fixture")" [[ "$after" != "$before" ]] || task13_fail "registry fingerprint ignored a new directory" linked="$fixture/linked" ln -s "$fixture/locks" "$linked" 2>/dev/null || return 0 if task13_registry_filesystem_fingerprint "$fixture" >/dev/null 2>&1; then task13_fail "registry fingerprint accepted a symbolic link" fi } task13_self_test_source_contract() { local root host_network push_command registry_function workflow uses_count pinned_uses_count local runner_preparation path local auth_runtime_mount auth_root_mount auth_projection auth_runtime_owner local pi_auth_bind pi_projection registry_runtime_mount registry_root_mount registry_projection local application_secret_bind application_secret_mount application_secret_projection local application_session_ca_fixture application_session_ca_source application_session_password_projection local application_secret_parent_owner application_secret_file_owner local image_evidence_environment image_evidence_initialization local server_auth_projection_override server_auth_projection_descriptor local server_auth_projection_environment server_auth_privileged_configure local server_fixture_reclamation server_runtime_config_probe server_workspace_config_permission local server_runtime_selector_probe server_runtime_generation_probe server_runtime_direct_file_probe local server_runtime_projection_status_contract local server_rollback_sentinel_read server_control_dir_reclamation local server_checkpoint_lookup local server_secret_source_owner server_secret_source_preparation server_tht_wrapper root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)" workflow="$root/.github/workflows/deployment.yml" runner_preparation="$root/scripts/prepare-linux-docker-runner.sh" host_network='--network'' host' push_command='docker image ''push' registry_function='task13_start_''registry' auth_runtime_mount='auth-runtime:/run/thothii-''auth:ro' auth_root_mount='$TASK13_AUTH_''ROOT:/run/thothii-auth:ro' auth_projection='task13_prepare_local_auth_''runtime' auth_runtime_owner='chown 10001:''10001 /target' pi_auth_bind='$TASK13_PI_''AUTH:/home/thoth/.pi/agent/auth.json:ro' pi_projection='task13_prepare_local_pi_''runtime' registry_runtime_mount='registry-remote:/fixtures/remote.git:''ro' registry_root_mount='$TASK13_''REMOTE:/fixtures/remote.git:ro' registry_projection='task13_prepare_registry_''remote' application_secret_bind='$TASK13_''SECRETS:/run/secrets/thothii.secrets:ro' application_secret_mount='application-secrets:/run/''secrets:ro' application_secret_projection='task13_prepare_local_application_''secrets' application_session_ca_fixture='task13_write_session_ca_''fixture' application_secret_parent_owner='chown 0:''0 /target' application_session_ca_source='$TASK13_SESSION_''CA:/source/session_ca.pem:ro' application_session_password_projection='cp /source/task13-runtime-password /target/session_''runtime_password' application_secret_file_owner='chown 10001:''10001 /target/thothii.secrets /target/task13-runtime-password /target/session_runtime_password /target/session_ca.pem' image_evidence_environment='TASK13_IMAGE_EVIDENCE_OUTPUT: ${{ runner.temp }}''/task13-images.json' image_evidence_initialization='TASK13_IMAGE_EVIDENCE_OUTPUT="${TASK13_IMAGE_EVIDENCE_OUTPUT:-$TASK13_ROOT/''.artifacts/task-15/unified-docker-images.json}"' server_auth_projection_override='deploy/compose.auth-runtime-''projection.yaml' server_auth_projection_descriptor='runtime''Projection:' server_auth_projection_environment='THT_AUTH_RUNTIME_''ROOT=%s' server_auth_privileged_configure='sudo -n -- "$TASK13_''THT"' server_fixture_reclamation='task13_reclaim_server_fixture_''ownership' server_runtime_config_probe='task13_compose_''logged "verify server runtime configuration and secret readability"' server_runtime_selector_probe='check_readable /run/thothii-auth/''CURRENT' server_runtime_generation_probe='check_readable "/run/thothii-auth/generations/$projection_generation/''auth.yaml"' server_runtime_direct_file_probe='check_readable /run/thothii-auth/''auth.yaml' server_runtime_projection_status_contract='value.state!=="''ready"||value.equal!==true' server_rollback_sentinel_read='sudo -n -- cat -- "$rollback_''sentinel"' server_control_dir_reclamation='"$TASK13_TMP" "$TASK13_CONTROL_''DIR"' server_checkpoint_lookup='task13_server_checkpoint_''leftover' server_secret_source_preparation='task13_prepare_server_secret_''sources' server_secret_source_owner='chown 10001:''10001 -- "$TASK13_SECRETS" "$TASK13_PI_AUTH"' server_tht_wrapper='task13_server_''tht' server_workspace_config_permission='chmod 0644 "$TASK13_SERVER_''WORKSPACE_CONFIG"' if rg -n 'docker[[:space:]]+(system[[:space:]]+)?prune' \ "$root/scripts/unified-deployment-smoke.sh" \ "$root/scripts/tht-update-smoke.sh" \ "$root/scripts/internal-semantic-smoke.sh" >/dev/null; then task13_fail "Task 13 smoke scripts must never prune global Docker state" fi ! grep -Fq -- "$host_network" "$root/scripts/unified-deployment-smoke.sh" \ || task13_fail "the image registry must not depend on host networking" if grep -Fq -- "$push_command" "$root/scripts/unified-deployment-smoke.sh" \ || grep -Fq -- "$registry_function" "$root/scripts/unified-deployment-smoke.sh"; then task13_fail "the rollback fixture must not depend on a daemon-to-host local image registry" fi grep -Fq -- "$auth_runtime_mount" "$root/scripts/unified-deployment-smoke.sh" \ || task13_fail "the local smoke must mount a Compose-owned authentication runtime volume" ! grep -Fq -- "$auth_root_mount" "$root/scripts/unified-deployment-smoke.sh" \ || task13_fail "the local smoke must not bind host-owned authentication into the core" [[ "$(grep -Ec "^${auth_projection}\\(\\)|^[[:space:]]+${auth_projection}$" \ "$root/scripts/unified-deployment-smoke.sh")" -eq 2 ]] \ || task13_fail "the local authentication runtime projection must be defined and invoked once" grep -Fq -- "$auth_runtime_owner" "$root/scripts/unified-deployment-smoke.sh" \ || task13_fail "the local authentication runtime projection must enforce the core UID" ! grep -Fq -- "$pi_auth_bind" "$root/scripts/unified-deployment-smoke.sh" \ || task13_fail "the local smoke must not bind host-owned Pi authentication into the core" [[ "$(grep -Ec "^${pi_projection}\\(\\)|^[[:space:]]+${pi_projection}$" \ "$root/scripts/unified-deployment-smoke.sh")" -eq 2 ]] \ || task13_fail "the local Pi runtime projection must be defined and invoked once" grep -Fq -- "$registry_runtime_mount" "$root/scripts/unified-deployment-smoke.sh" \ || task13_fail "the local smoke must mount a Compose-owned Git fixture volume" ! grep -Fq -- "$registry_root_mount" "$root/scripts/unified-deployment-smoke.sh" \ || task13_fail "the local smoke must not bind the host-owned Git fixture into the core" [[ "$(grep -Ec "^${registry_projection}\\(\\)|^[[:space:]]+${registry_projection}$" \ "$root/scripts/unified-deployment-smoke.sh")" -ge 3 ]] \ || task13_fail "the Git fixture projection must cover bootstrap and subsequent pushes" ! grep -Fq -- "$application_secret_bind" "$root/scripts/unified-deployment-smoke.sh" \ || task13_fail "the local smoke must not bind the host-owned application bundle into the core" grep -Fq -- "$application_secret_mount" "$root/scripts/unified-deployment-smoke.sh" \ || task13_fail "the local smoke must mount Compose-owned application secrets" [[ "$(grep -Ec "^${application_secret_projection}\\(\\)|^[[:space:]]+${application_secret_projection}$" \ "$root/scripts/unified-deployment-smoke.sh")" -eq 2 ]] \ || task13_fail "the local application-secret projection must be defined and invoked once" [[ "$(grep -Ec "^${application_session_ca_fixture}\\(\\)|^[[:space:]]+${application_session_ca_fixture}$" \ "$root/scripts/unified-deployment-smoke.sh")" -eq 3 ]] \ || task13_fail "the session CA fixture must be defined and written for local and server smokes" grep -Fq -- "$application_secret_parent_owner" "$root/scripts/unified-deployment-smoke.sh" \ || task13_fail "the local application-secret mount root must remain root-owned" grep -Fq -- "$application_session_ca_source" "$root/scripts/unified-deployment-smoke.sh" \ || task13_fail "the local application-secret projection must include the session CA" grep -Fq -- "$application_session_password_projection" "$root/scripts/unified-deployment-smoke.sh" \ || task13_fail "the local application-secret projection must expose the session password name" grep -Fq -- "$application_secret_file_owner" "$root/scripts/unified-deployment-smoke.sh" \ || task13_fail "the projected application secrets must remain readable only by the core UID" grep -Eq '^TASK13_BAD_CANDIDATE_IMAGE="[^"[:space:]]+@sha256:[0-9a-f]{64}"$' \ "$root/scripts/unified-deployment-smoke.sh" \ || task13_fail "the bad rollback candidate must be an immutable digest reference" grep -Eq 'timeout .*scripts/unified-deployment-smoke\.sh' "$workflow" \ || task13_fail "CI lacks an outer timeout for the unified deployment smoke" grep -Eq 'timeout .*scripts/tht-update-smoke\.sh' "$workflow" \ || task13_fail "CI lacks an outer timeout for the tht update smoke" grep -Fq 'bash scripts/prepare-linux-docker-runner.sh' "$workflow" \ || task13_fail "CI must reclaim unused hosted-runner toolchains before the Docker release smoke" [[ "$(grep -Fc -- "$image_evidence_environment" "$workflow")" -eq 3 ]] \ || task13_fail "CI must write generated Docker image evidence outside the trusted checkout" grep -Fq -- "$image_evidence_initialization" "$root/scripts/unified-deployment-smoke.sh" \ || task13_fail "the Docker image evidence output must honor an explicit CI path" grep -Fq -- "$server_auth_projection_override" "$root/scripts/unified-deployment-smoke.sh" \ || task13_fail "the server smoke must mount the UID 10001 authentication projection" grep -Fq -- "$server_auth_projection_descriptor" "$root/scripts/unified-deployment-smoke.sh" \ || task13_fail "the server smoke installation must declare its authentication projection" grep -Fq -- "$server_auth_projection_environment" "$root/scripts/unified-deployment-smoke.sh" \ || task13_fail "the server smoke must export the authentication projection root" grep -Fq -- "$server_auth_privileged_configure" "$root/scripts/unified-deployment-smoke.sh" \ || task13_fail "the server smoke must publish projected authentication as root" [[ "$(grep -Ec "^${server_fixture_reclamation}\\(\\)|^[[:space:]]+${server_fixture_reclamation}$" \ "$root/scripts/unified-deployment-smoke.sh")" -eq 2 ]] \ || task13_fail "the server smoke must reclaim its root- and core-owned fixture exactly once" grep -Fq -- "$server_runtime_config_probe" "$root/scripts/unified-deployment-smoke.sh" \ || task13_fail "the server runtime preconditions must emit a named diagnostic" grep -Fq -- "$server_runtime_selector_probe" "$root/scripts/unified-deployment-smoke.sh" \ || task13_fail "the server runtime preconditions must verify the projection selector" grep -Fq -- "$server_runtime_generation_probe" "$root/scripts/unified-deployment-smoke.sh" \ || task13_fail "the server runtime preconditions must verify the selected generation" ! grep -Fq -- "$server_runtime_direct_file_probe" "$root/scripts/unified-deployment-smoke.sh" \ || task13_fail "the server runtime preconditions must not require the forbidden direct-file fallback" grep -Fq -- "$server_runtime_projection_status_contract" "$root/scripts/unified-deployment-smoke.sh" \ || task13_fail "the server status assertion must validate projection readiness" grep -Fq -- "$server_rollback_sentinel_read" "$root/scripts/unified-deployment-smoke.sh" \ || task13_fail "the server rollback sentinel must be read through the privileged host surface" grep -Fq -- "$server_control_dir_reclamation" "$root/scripts/unified-deployment-smoke.sh" \ || task13_fail "server cleanup must reclaim both temporary and control roots" [[ "$(grep -Ec "^${server_checkpoint_lookup}\\(\\)|${server_checkpoint_lookup}" \ "$root/scripts/unified-deployment-smoke.sh")" -eq 3 ]] \ || task13_fail "server restore must inspect root-owned checkpoints through one privileged helper" grep -Fq -- "$server_workspace_config_permission" "$root/scripts/unified-deployment-smoke.sh" \ || task13_fail "the server workspace fixture must be readable by the container UID" [[ "$(grep -Ec "^${server_secret_source_preparation}\\(\\)|^[[:space:]]+${server_secret_source_preparation}$" \ "$root/scripts/unified-deployment-smoke.sh")" -eq 2 ]] \ || task13_fail "the server secret source preparation must be defined and invoked once" grep -Fq -- "$server_secret_source_owner" "$root/scripts/unified-deployment-smoke.sh" \ || task13_fail "the server secret sources must be private and owned by the container UID" [[ "$(grep -Fc -- "$server_tht_wrapper" "$root/scripts/unified-deployment-smoke.sh")" -eq 10 ]] \ || task13_fail "server operator commands must use the privileged canonical-auth wrapper" [[ -x "$runner_preparation" ]] \ || task13_fail "the Linux Docker runner preparation must be executable" for path in /usr/local/lib/android /usr/share/dotnet /opt/ghc /usr/local/.ghcup; do grep -Fq -- "$path" "$runner_preparation" \ || task13_fail "the Linux Docker runner preparation is missing the scoped path: $path" done ! grep -Eq '/opt/hostedtoolcache([/[:space:]]|$)|rm[[:space:]]+-rf[[:space:]]+--?[[:space:]]+/($|[[:space:]])' "$runner_preparation" \ || task13_fail "the Linux Docker runner preparation must not remove required tool caches or broad roots" uses_count="$(grep -Ec '^[[:space:]]+uses:' "$workflow")" pinned_uses_count="$(grep -Ec '^[[:space:]]+uses: [^[:space:]]+@[0-9a-f]{40}([[:space:]]|$)' "$workflow")" [[ "$uses_count" -gt 0 && "$uses_count" -eq "$pinned_uses_count" ]] \ || task13_fail "every deployment workflow action must use a full immutable commit pin" if grep -Fq '${{ secrets.' "$workflow"; then task13_fail "the deployment release gate must not require repository secrets" fi for command in \ 'bash scripts/verify-line-endings.sh' \ 'bash scripts/test-unified-compose.sh' \ 'bash scripts/test-compose-secret-policy.sh' \ 'bash scripts/test-no-deployment-coupling.sh' \ 'bash scripts/test-verify-workspace-install-docs.sh' \ 'npx vitest run' \ 'npx tsc --noEmit -p .' \ 'npx tsc -b' \ './scripts/test-windows-clone-contract.ps1'; do grep -Fq "$command" "$workflow" || task13_fail "CI coverage is missing: $command" done } task13_self_test() { task13_self_test_sanitizer task13_self_test_core_startup_diagnostics task13_self_test_server_workspace_diagnostics task13_self_test_cleanup_ownership task13_self_test_image_cleanup_ownership task13_self_test_transaction_image_cleanup task13_self_test_image_evidence task13_self_test_rollback_fixture_contract task13_self_test_runtime_binding_fixture task13_self_test_server_runtime_binding_fixture task13_self_test_stopped_project_containers task13_self_test_timeout_process_group task13_self_test_nested_timeout_process_group task13_self_test_public_timeout_contract task13_self_test_internal_semantic_offline_contract task13_self_test_windows_release_contract task13_self_test_server_release_contract task13_self_test_registry_fingerprint task13_self_test_source_contract printf 'Task 13 smoke safety contracts passed.\n' } task13_self_test_case() { case "$1" in rollback) task13_self_test_rollback_fixture_contract ;; runtime-bindings) task13_self_test_runtime_binding_fixture ;; server-bindings) task13_self_test_server_runtime_binding_fixture ;; cleanup) task13_self_test_stopped_project_containers ;; image-evidence) task13_self_test_image_evidence ;; timeout-group) task13_self_test_timeout_process_group ;; timeout-nested) task13_self_test_nested_timeout_process_group ;; timeout-public) task13_self_test_public_timeout_contract ;; semantic-offline) task13_self_test_internal_semantic_offline_contract ;; windows) task13_self_test_windows_release_contract ;; server) task13_self_test_server_release_contract ;; server-diagnostics) task13_self_test_server_workspace_diagnostics ;; startup-diagnostics) task13_self_test_core_startup_diagnostics ;; *) task13_fail "unknown Task 13 self-test case: $1" ;; esac } task13_fixtures_only() { local tmp descriptor_path catalog_path initial_head updated_head tmp="$(mktemp -d "${TMPDIR:-/tmp}/thothii-task13-fixtures.XXXXXX")" trap 'rm -rf "$tmp"' RETURN TASK13_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)" TASK13_TMP="$tmp" TASK13_REMOTE="$tmp/remote.git" TASK13_SEED="$tmp/seed" TASK13_BRANCH="task13-smoke" TASK13_LOG="$tmp/task13.log" TASK13_FAILURE_LOGGED=0 : >"$TASK13_LOG" task13_seed_registry descriptor_path="$TASK13_SEED/$TASK13_WORKSPACE_ID/workspace.yaml" catalog_path="$TASK13_SEED/thoth-workspaces.yaml" [[ -f "$catalog_path" ]] || task13_fail "missing Task 13 root workspace catalog" [[ -f "$descriptor_path" ]] || task13_fail "missing Task 13 nested workspace descriptor" [[ -f "$TASK13_SEED/$TASK13_WORKSPACE_ID/evidence/guide.md" ]] || task13_fail "missing Task 13 nested workspace evidence" [[ ! -e "$TASK13_SEED/workspaces/$TASK13_WORKSPACE_ID.yaml" ]] || task13_fail "legacy Task 13 flat descriptor path exists" [[ ! -e "$TASK13_SEED/workspace-content/$TASK13_WORKSPACE_ID" ]] || task13_fail "legacy Task 13 flat evidence path exists" initial_head="$(git -C "$TASK13_SEED" rev-parse HEAD)" task13_replace_once "$descriptor_path" 'name: Task 13 Smoke' 'name: Task 13 Smoke Updated' task13_replace_once "$catalog_path" 'name: Task 13 Smoke' 'name: Task 13 Smoke Updated' task13_commit_registry_change 'Update Task 13 workspace metadata' updated_head="$(git -C "$TASK13_SEED" rev-parse HEAD)" [[ "$updated_head" != "$initial_head" ]] || task13_fail "Task 13 metadata update did not advance Git head" printf 'guide v2\n' >"$TASK13_SEED/$TASK13_WORKSPACE_ID/evidence/guide.md" task13_commit_registry_change 'Update Task 13 workspace evidence only' [[ "$(git -C "$TASK13_SEED" rev-parse HEAD)" != "$updated_head" ]] || task13_fail "Task 13 evidence-only commit did not advance Git head" mkdir -p "$TASK13_SEED/workspaces" "$TASK13_SEED/workspace-content/$TASK13_WORKSPACE_ID/evidence" cp "$TASK13_ROOT/scripts/fixtures/workspace-registry-task13.yaml" "$TASK13_SEED/workspaces/$TASK13_WORKSPACE_ID.yaml" printf 'legacy guide\n' >"$TASK13_SEED/workspace-content/$TASK13_WORKSPACE_ID/evidence/guide.md" [[ -f "$TASK13_SEED/workspaces/$TASK13_WORKSPACE_ID.yaml" ]] || task13_fail "missing Task 13 legacy flat descriptor fixture" [[ -f "$TASK13_SEED/workspace-content/$TASK13_WORKSPACE_ID/evidence/guide.md" ]] || task13_fail "missing Task 13 legacy flat evidence fixture" printf 'Task 13 fixture contract passed. ' } task13_initialize() { local source_status umask 077 TASK13_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)" TASK13_TMP_PARENT="$(cd "${TMPDIR:-/tmp}" && pwd -P)" TASK13_TMP="$(mktemp -d "$TASK13_TMP_PARENT/thothii-task13.XXXXXX")" TASK13_TMP="$(cd "$TASK13_TMP" && pwd -P)" TASK13_LOG="$TASK13_TMP/task13.log" TASK13_FAILURE_LOGGED=0 : >"$TASK13_LOG" trap 'task13_cleanup $?' EXIT trap 'exit 130' INT TERM HUP TASK13_RUN_ID="$(date -u +%Y%m%d%H%M%S)-$$-${RANDOM:-0}" TASK13_SOURCE_COMMIT="$(git -C "$TASK13_ROOT" rev-parse --verify HEAD)" [[ "$TASK13_SOURCE_COMMIT" =~ ^[0-9a-f]{40}$ ]] || task13_fail "source commit was not resolved" source_status="$(git -C "$TASK13_ROOT" status --porcelain --untracked-files=normal \ | sed -e '/^?? \.playwright-cli\/$/d' -e '/^?? \.thothctl\/$/d')" [[ -z "$source_status" ]] || task13_fail "source must be clean for image traceability" TASK13_IMAGE_EVIDENCE_RECORDS="$TASK13_TMP/image-evidence.tsv" : >"$TASK13_IMAGE_EVIDENCE_RECORDS" TASK13_IMAGE_EVIDENCE_OUTPUT="${TASK13_IMAGE_EVIDENCE_OUTPUT:-$TASK13_ROOT/.artifacts/task-15/unified-docker-images.json}" [[ "$TASK13_IMAGE_EVIDENCE_OUTPUT" = /* ]] \ || task13_fail "Docker image evidence output must be an absolute path" rm -f "$TASK13_IMAGE_EVIDENCE_OUTPUT" TASK13_PROFILE="local" TASK13_INSTALLATION="$TASK13_TMP/thothii-installation.yaml" TASK13_PROJECT="thothii-$(task13_sha256_text "$TASK13_INSTALLATION" | cut -c1-12)" TASK13_CONTROL_DIR="$TASK13_ROOT/.tht/$TASK13_PROJECT" [[ ! -e "$TASK13_CONTROL_DIR" ]] || task13_fail "unique tht control directory already exists" TASK13_CURRENT_IMAGE_OVERRIDE="$TASK13_CONTROL_DIR/current-image.yaml" TASK13_UPDATE_STATE="$TASK13_CONTROL_DIR/update-state.json" TASK13_REMOTE="$TASK13_TMP/remote.git" TASK13_SEED="$TASK13_TMP/seed" TASK13_BRANCH="task13-smoke" TASK13_ENV_FILE="$TASK13_TMP/local.env" TASK13_OVERRIDE="$TASK13_TMP/compose.task13.yaml" TASK13_PI_AUTH="$TASK13_TMP/pi-auth.json" TASK13_SECRETS="$TASK13_TMP/thothii.secrets" TASK13_AUTH_ROOT="$TASK13_TMP/auth" TASK13_AUTH_RUNTIME_ROOT="$TASK13_TMP/auth-runtime" TASK13_AUTH_PASSWORD_FILE="$TASK13_TMP/local-auth-password" TASK13_AUTH_RUNTIME_VOLUME="${TASK13_PROJECT}_auth-runtime" TASK13_AUTH_PROJECTION_CONTAINER="$TASK13_PROJECT-auth-projection" TASK13_PI_RUNTIME_VOLUME="${TASK13_PROJECT}_pi-state" TASK13_PI_PROJECTION_CONTAINER="$TASK13_PROJECT-pi-projection" TASK13_APPLICATION_SECRETS_VOLUME="${TASK13_PROJECT}_application-secrets" TASK13_APPLICATION_SECRETS_PROJECTION_CONTAINER="$TASK13_PROJECT-application-secrets-projection" TASK13_REGISTRY_RUNTIME_VOLUME="${TASK13_PROJECT}_registry-remote" TASK13_REGISTRY_PROJECTION_CONTAINER="$TASK13_PROJECT-registry-projection" TASK13_AUTH_ADMIN=task13-admin TASK13_AUTH_PASSWORD="task13-auth-$TASK13_RUN_ID" TASK13_OIDC_CLIENT_SECRET="task13-oidc-client-$TASK13_RUN_ID" TASK13_AUTHENTIK_API_TOKEN="task13-authentik-token-$TASK13_RUN_ID" TASK13_PI_MODELS="$TASK13_TMP/models.json" TASK13_PI_SETTINGS="$TASK13_TMP/pi-settings.json" TASK13_LLM_SERVER="$TASK13_TMP/fake-llm.mjs" TASK13_OIDC_SERVER="$TASK13_TMP/fake-oidc.mjs" TASK13_OIDC_CERT="$TASK13_TMP/fake-oidc-cert.pem" TASK13_OIDC_KEY="$TASK13_TMP/fake-oidc-key.pem" TASK13_THT_DIR="$TASK13_TMP/tht" TASK13_LLM_CONTAINER="$TASK13_PROJECT-llm" TASK13_OIDC_CONTAINER="$TASK13_PROJECT-oidc" TASK13_BAD_CANDIDATE_CONTAINER="$TASK13_PROJECT-bad-candidate" TASK13_CORE_IMAGE="task13-core-$TASK13_RUN_ID:local" TASK13_FRONTEND_IMAGE="task13-frontend-$TASK13_RUN_ID:local" TASK13_SECRET_VALUE="task13-secret-$TASK13_RUN_ID" TASK13_NETWORK="" TASK13_BAD_CANDIDATE_ID="" TASK13_PREVIOUS_IMAGE_ID="" TASK13_SERVER_DATA="$TASK13_TMP/Server Data" TASK13_SERVER_PI_STATE="$TASK13_TMP/Server Pi State" TASK13_SERVER_REGISTRY="$TASK13_TMP/Server Registry" TASK13_SERVER_WORKSPACE_CONFIG="$TASK13_TMP/server-sessions.yaml" TASK13_SESSION_RUNTIME_PASSWORD="$TASK13_TMP/session-runtime-password" TASK13_SESSION_MIGRATOR_PASSWORD_FILE="$TASK13_TMP/session-migrator-password" TASK13_SESSION_CA="$TASK13_TMP/session-ca.pem" TASK13_SESSION_PASSWORD="task13-runtime-$TASK13_RUN_ID" TASK13_SESSION_MIGRATOR_PASSWORD="task13-migrator-$TASK13_RUN_ID" TASK13_FRONTEND_PORT="" } task13_require_tools() { for command in bash git docker curl node openssl python3 sed awk grep rg sort; do command -v "$command" >/dev/null 2>&1 || task13_fail "$command is required" done if [[ "${TASK13_PROFILE:-local}" == server ]]; then command -v sudo >/dev/null 2>&1 || task13_fail "sudo is required for the Linux server smoke" sudo -n -- true >/dev/null 2>&1 \ || task13_fail "passwordless sudo is required for the Linux server smoke" fi task13_run_logged "Docker daemon readiness" docker info task13_run_logged "Docker Compose readiness" docker compose version task13_self_test_source_contract } task13_smoke_main() { local mode="${1:-full}" [[ "$mode" == full || "$mode" == update ]] || task13_fail "unknown Task 13 smoke mode: $mode" task13_initialize task13_require_tools TASK13_FRONTEND_PORT="$(node -e 'const net=require("node:net"); const server=net.createServer(); server.listen(0,"127.0.0.1",()=>{process.stdout.write(String(server.address().port)); server.close()})')" [[ "$TASK13_FRONTEND_PORT" =~ ^[1-9][0-9]*$ ]] || task13_fail "could not reserve a loopback frontend port" task13_write_fixture_files task13_write_environment /fixtures/remote.git task13_seed_registry task13_build_tht task13_configure_local_authentication task13_start_stack task13_record_project_image_evidence task13_record_image_evidence "$TASK13_CORE_IMAGE" fixture-runtime task13_assert_project_ownership task13_assert_built_image_ownership task13_assert_runtime task13_assert_local_restore_reauthentication task13_prepare_persistence if [[ "$mode" == full ]]; then task13_registry_lifecycle fi task13_update_rollback task13_record_project_image_evidence TASK13_IMAGE_EVIDENCE_STATUS=pass task13_write_image_evidence printf 'Task 13 %s deployment smoke passed.\n' "$mode" } task13_server_smoke_main() { task13_initialize TASK13_PROFILE="server" task13_require_tools task13_write_server_fixture_files task13_seed_registry task13_build_tht task13_prepare_server_auth_roots task13_prepare_server_secret_sources task13_configure_server_oidc_authentication task13_start_server_stack task13_record_project_image_evidence task13_record_image_evidence "$TASK13_CORE_IMAGE" fixture-runtime task13_assert_project_ownership task13_assert_built_image_ownership task13_assert_server_runtime task13_assert_server_oidc_restore_verification task13_record_project_image_evidence TASK13_IMAGE_EVIDENCE_STATUS=pass task13_write_image_evidence printf 'Task 13 Linux server deployment smoke passed.\n' } if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then if [[ "${1:-}" == "--self-test" ]]; then task13_self_test elif [[ "${1:-}" == "--self-test-case" ]]; then [[ -n "${2:-}" ]] || task13_fail "--self-test-case requires a case name" task13_self_test_case "$2" elif [[ "${1:-}" == "--fixtures-only" ]]; then task13_fixtures_only else task13_supervise "$TASK13_SMOKE_TIMEOUT" "unified deployment smoke" task13_smoke_main full fi fi