mptyl/ThothII
1
0
Fork 0
Code Issues 25 Pull Requests Actions Packages Projects Releases 1 Wiki Activity
Files
main
ThothII/scripts/unified-deployment-smoke.sh
T

3129 lines
143 KiB
Bash
Executable File
Raw Permalink Blame History

#!/usr/bin/env bash
# End-to-end release gate for the canonical two-service Compose distribution.
# This file is also sourced by tht-update-smoke.sh so both entry points use the same
# isolated fixture, exact cleanup, and sanitized failure reporting.
set -euo pipefail
TASK13_BAD_CANDIDATE_IMAGE="hello-world@sha256:c3cbe1cc1aa588a64951ac6286e0df7b27fe2e6324b1001c619bb358770c0178"
TASK13_BAD_CANDIDATE_BEHAVIOR="stopped"
TASK13_BAD_PI_VERSION="0.80.4-task13"
TASK13_CURL_CONNECT_TIMEOUT=3
TASK13_CURL_MAX_TIME=10
TASK13_CLEANUP_TIMEOUT=20
TASK13_COMMAND_TIMEOUT=900
TASK13_SMOKE_TIMEOUT=1800
TASK13_TERM_GRACE=45
task13_fail() {
printf 'Task 13 smoke failed: %s\n' "$*" >&2
return 1
}
task13_sha256_text() {
if command -v sha256sum >/dev/null 2>&1; then
printf '%s' "$1" | sha256sum | awk '{print $1}'
elif command -v shasum >/dev/null 2>&1; then
printf '%s' "$1" | shasum -a 256 | awk '{print $1}'
else
task13_fail "sha256sum or shasum is required"
fi
}
task13_record_image_evidence() {
local reference="$1" role="$2" image_id verified_id repo_digests
[[ "$role" =~ ^[a-z0-9-]+$ ]] || task13_fail "invalid image evidence role"
image_id="$(docker image inspect --format '{{.Id}}' "$reference")"
[[ "$image_id" =~ ^sha256:[0-9a-f]{64}$ ]] \
|| task13_fail "Docker image identity was not resolved"
verified_id="$(docker image inspect --format '{{.Id}}' "$image_id")"
[[ "$verified_id" == "$image_id" ]] \
|| task13_fail "Docker image identity changed during evidence capture"
if ! repo_digests="$({ docker image inspect --format '{{json .RepoDigests}}' "$image_id"; } | node -e '
const fs = require("node:fs");
let value = JSON.parse(fs.readFileSync(0, "utf8"));
if (value === null) value = [];
if (!Array.isArray(value)) process.exit(1);
const digests = [];
for (const item of value) {
if (typeof item !== "string" || !/@sha256:[0-9a-f]{64}$/.test(item)) process.exit(1);
digests.push(item.slice(item.lastIndexOf("@") + 1));
}
process.stdout.write(JSON.stringify([...new Set(digests)].sort()));
')"; then
task13_fail "Docker repository digest evidence was invalid"
return 1
fi
printf '%s\t%s\t%s\n' "$image_id" "$role" "$repo_digests" >>"$TASK13_IMAGE_EVIDENCE_RECORDS"
}
task13_record_project_image_evidence() {
local container_id image_id count=0
while IFS= read -r container_id; do
[[ -n "$container_id" ]] || continue
image_id="$(docker container inspect --format '{{.Image}}' "$container_id")"
task13_record_image_evidence "$image_id" compose-runtime
count=$((count + 1))
done < <(docker container ls -aq --filter "label=com.docker.compose.project=$TASK13_PROJECT")
[[ "$count" -gt 0 ]] || task13_fail "no Compose runtime images were available for evidence capture"
}
task13_write_image_evidence() {
local image_id verified_id output_dir temporary
while IFS= read -r image_id; do
[[ -n "$image_id" ]] || continue
verified_id="$(docker image inspect --format '{{.Id}}' "$image_id")"
[[ "$verified_id" == "$image_id" ]] \
|| task13_fail "Docker image identity was unavailable before cleanup"
done < <(cut -f1 "$TASK13_IMAGE_EVIDENCE_RECORDS" | LC_ALL=C sort -u)
output_dir="$(dirname "$TASK13_IMAGE_EVIDENCE_OUTPUT")"
mkdir -p "$output_dir"
temporary="$(mktemp "$output_dir/.unified-docker-images.XXXXXX")"
if ! node - "$TASK13_IMAGE_EVIDENCE_RECORDS" "$TASK13_SOURCE_COMMIT" "$TASK13_RUN_ID" \
"$TASK13_IMAGE_EVIDENCE_STATUS" >"$temporary" <<'NODE'
const fs = require("node:fs");
const [recordsFile, sourceCommit, runId, status] = process.argv.slice(2);
if (!/^[0-9a-f]{40}$/.test(sourceCommit) || !/^[0-9A-Za-z-]+$/.test(runId)
|| !/^(?:pass|fail)$/.test(status)) process.exit(1);
const images = new Map();
for (const line of fs.readFileSync(recordsFile, "utf8").split("\n").filter(Boolean)) {
const fields = line.split("\t");
if (fields.length !== 3) process.exit(1);
const [id, role, encodedDigests] = fields;
if (!/^sha256:[0-9a-f]{64}$/.test(id) || !/^[a-z0-9-]+$/.test(role)) process.exit(1);
const repoDigests = JSON.parse(encodedDigests);
if (!Array.isArray(repoDigests)
|| repoDigests.some((digest) => typeof digest !== "string" || !/^sha256:[0-9a-f]{64}$/.test(digest))) {
process.exit(1);
}
const current = images.get(id) ?? { id, roles: new Set(), repo_digests: new Set() };
current.roles.add(role);
for (const digest of repoDigests) current.repo_digests.add(digest);
images.set(id, current);
}
if (images.size === 0) process.exit(1);
const document = {
gate: "unified-deployment-smoke",
status,
source_commit: sourceCommit,
run_id: runId,
images: [...images.values()].sort((left, right) => left.id.localeCompare(right.id)).map((image) => ({
id: image.id,
roles: [...image.roles].sort(),
repo_digests: [...image.repo_digests].sort(),
})),
};
process.stdout.write(`${JSON.stringify(document, null, 2)}\n`);
NODE
then
rm -f "$temporary"
task13_fail "could not serialize sanitized Docker image evidence"
return 1
fi
chmod 0600 "$temporary"
mv "$temporary" "$TASK13_IMAGE_EVIDENCE_OUTPUT"
}
task13_capture_failed_image_evidence() {
TASK13_IMAGE_EVIDENCE_STATUS=fail
task13_record_project_image_evidence
task13_write_image_evidence
}
task13_registry_filesystem_fingerprint() {
local root="$1"
local -a python=(python3)
if [[ "${TASK13_PROFILE:-local}" == server && "$root" == "${TASK13_SERVER_REGISTRY:-}" ]]; then
[[ -n "${TASK13_TMP:-}" \
&& "$root" == "$TASK13_TMP/Server Registry" \
&& ! -L "$root" && -d "$root" ]] \
|| task13_fail "refusing to fingerprint an unexpected server registry path"
python=(sudo -n -- python3)
fi
"${python[@]}" - "$root" <<'PY'
import hashlib
import os
import stat
import sys
root = os.path.abspath(sys.argv[1])
records = []
entries = 0
total_bytes = 0
def snapshot(value):
return (
value.st_dev, value.st_ino, value.st_mode, value.st_uid, value.st_gid,
value.st_size, value.st_mtime_ns, value.st_ctime_ns,
)
def visit(path, relative):
global entries, total_bytes
before = os.lstat(path)
entries += 1
if entries > 65536:
raise SystemExit("registry fingerprint entry bound exceeded")
metadata = snapshot(before)
if stat.S_ISLNK(before.st_mode):
raise SystemExit("registry fingerprint rejected a symbolic link")
if stat.S_ISDIR(before.st_mode):
records.append(("directory", relative, metadata))
with os.scandir(path) as listing:
children = sorted((item.name for item in listing))
for name in children:
visit(os.path.join(path, name), name if relative == "." else relative + "/" + name)
elif stat.S_ISREG(before.st_mode):
if before.st_size > 256 * 1024 * 1024:
raise SystemExit("registry fingerprint file bound exceeded")
total_bytes += before.st_size
if total_bytes > 2 * 1024 * 1024 * 1024:
raise SystemExit("registry fingerprint total bound exceeded")
flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0)
descriptor = os.open(path, flags)
try:
opened = os.fstat(descriptor)
if snapshot(opened) != metadata:
raise SystemExit("registry changed while fingerprinting")
digest = hashlib.sha256()
while True:
chunk = os.read(descriptor, 1024 * 1024)
if not chunk:
break
digest.update(chunk)
finally:
os.close(descriptor)
records.append(("file", relative, metadata, digest.hexdigest()))
else:
raise SystemExit("registry fingerprint rejected a special file")
if snapshot(os.lstat(path)) != metadata:
raise SystemExit("registry changed while fingerprinting")
visit(root, ".")
encoded = repr(records).encode("utf-8")
print("sha256:" + hashlib.sha256(encoded).hexdigest())
PY
}
task13_sanitize() {
local line
while IFS= read -r line || [[ -n "$line" ]]; do
if [[ -n "${TASK13_SECRET_VALUE:-}" ]]; then
line="${line//"$TASK13_SECRET_VALUE"/[REDACTED]}"
fi
printf '%s\n' "$line"
done | sed -E \
-e 's#([[:alpha:]][[:alnum:]+.-]*://[^:/@[:space:]]+:)[^@/[:space:]]+@#\1[REDACTED]@#g' \
-e 's/(([Pp]roxy-)?[Aa]uthorization:[[:space:]]*([Bb]earer|[Bb]asic)[[:space:]]+)[^[:space:]]+/\1[REDACTED]/g' \
-e "s/(([\"']?[[:alnum:]_.-]*(password|token|api[_-]?key|secret|key)[[:alnum:]_.-]*[\"']?[[:space:]]*[:=][[:space:]]*)([\"'][^\"']*[\"']|[^[:space:],;]+))/\2[REDACTED]/Ig"
}
task13_log_failure() {
local label="$1"
TASK13_FAILURE_LOGGED=1
printf 'Task 13 command failed: %s\n' "$label" >&2
tail -n 200 "$TASK13_LOG" | task13_sanitize >&2
return 1
}
task13_run_logged() {
local label="$1"
shift
if ! task13_bounded "$TASK13_COMMAND_TIMEOUT" "$label" "$@" >>"$TASK13_LOG" 2>&1; then
task13_log_failure "$label"
fi
}
task13_bounded() {
local seconds="$1" label="$2" command_pid watchdog_pid rc watchdog_rc=0
local monitor_enabled=0 timeout_marker command_group
shift 2
timeout_marker="$(mktemp "${TMPDIR:-/tmp}/thothii-task13-timeout.XXXXXX")"
[[ $- == *m* ]] && monitor_enabled=1
if [[ -n "${TASK13_ACTIVE_GROUP:-}" ]]; then
[[ "$TASK13_ACTIVE_GROUP" =~ ^[0-9]+$ ]] \
|| task13_fail "invalid active Task 13 process group"
set +m
"$@" &
command_pid=$!
command_group="$TASK13_ACTIVE_GROUP"
else
set -m
"$@" &
command_pid=$!
command_group="$command_pid"
[[ "$monitor_enabled" -eq 1 ]] || set +m
fi
(
local sleep_pid grace_deadline
sleep "$seconds" &
sleep_pid=$!
trap 'kill "$sleep_pid" 2>/dev/null || true' EXIT INT TERM
wait "$sleep_pid" 2>/dev/null || exit 0
trap - EXIT INT TERM
if kill -0 -- "-$command_group" 2>/dev/null; then
trap '' TERM
printf 'timeout\n' >"$timeout_marker"
printf 'Task 13 command timed out after %ss: %s\n' "$seconds" "$label" >&2
kill -TERM -- "-$command_group" 2>/dev/null || true
grace_deadline=$((SECONDS + TASK13_TERM_GRACE))
while kill -0 -- "-$command_group" 2>/dev/null && ((SECONDS < grace_deadline)); do
sleep 1
done
kill -KILL -- "-$command_group" 2>/dev/null || true
exit 124
fi
) &
watchdog_pid=$!
if [[ -n "${TASK13_ACTIVE_GROUP:-}" && "$monitor_enabled" -eq 1 ]]; then
set -m
fi
if wait "$command_pid"; then
rc=0
else
rc=$?
fi
if [[ -s "$timeout_marker" ]]; then
wait "$watchdog_pid" 2>/dev/null || watchdog_rc=$?
else
kill "$watchdog_pid" 2>/dev/null || true
wait "$watchdog_pid" 2>/dev/null || true
fi
rm -f "$timeout_marker"
[[ "$watchdog_rc" -eq 124 ]] && return 124
return "$rc"
}
task13_supervised_call() {
TASK13_ACTIVE_GROUP="$BASHPID"
"$@"
}
task13_supervise() {
local seconds="$1" label="$2"
shift 2
task13_bounded "$seconds" "$label" task13_supervised_call "$@"
}
task13_compose_files() {
TASK13_COMPOSE=(
docker compose
--project-name "$TASK13_PROJECT"
--project-directory "$TASK13_ROOT"
--env-file "$TASK13_ENV_FILE"
-f "$TASK13_ROOT/compose.yaml"
)
if [[ "${TASK13_PROFILE:-local}" == server ]]; then
TASK13_COMPOSE+=(
-f "$TASK13_ROOT/deploy/compose.server.yaml"
-f "$TASK13_ROOT/deploy/compose.session-server.yaml.example"
-f "$TASK13_OVERRIDE"
-f "${TASK13_INSTALLATION%/*}/generated/compose.models.yaml"
-f "$TASK13_ROOT/deploy/compose.auth-runtime-projection.yaml"
)
else
TASK13_COMPOSE+=(
-f "$TASK13_ROOT/deploy/compose.local.yaml"
-f "$TASK13_OVERRIDE"
-f "${TASK13_INSTALLATION%/*}/generated/compose.models.yaml"
)
fi
if [[ "${TASK13_PROFILE:-local}" == local && -f "$TASK13_CURRENT_IMAGE_OVERRIDE" ]]; then
TASK13_COMPOSE+=(-f "$TASK13_CURRENT_IMAGE_OVERRIDE")
fi
}
task13_compose_invoke() {
if [[ "${TASK13_PROFILE:-local}" == server ]]; then
sudo -n -- "$@"
else
"$@"
fi
}
task13_compose() {
task13_compose_files
task13_compose_invoke "${TASK13_COMPOSE[@]}" "$@"
}
task13_compose_logged() {
local label="$1"
shift
task13_compose_files
task13_run_logged "$label" task13_compose_invoke "${TASK13_COMPOSE[@]}" "$@"
}
task13_report_core_startup_failure() {
local container_id state exit_code logs cause="startup failure is unclassified"
container_id="$(task13_compose ps --all -q core 2>/dev/null | head -n 1 || true)"
state=""
if [[ -n "$container_id" ]]; then
state="$(docker inspect --format '{{.State.Status}}:{{.State.ExitCode}}' "$container_id" 2>/dev/null || true)"
fi
exit_code="${state##*:}"
[[ "$exit_code" =~ ^[0-9]{1,3}$ ]] || exit_code="unknown"
logs="$(task13_compose logs --no-color --tail 100 core 2>/dev/null || true)"
case "$logs" in
*auth_session_store_invalid*|*auth*storage*request*failed*|*EACCES*auth*|*permission*auth*)
cause="authentication state storage is unavailable"
;;
*auth_config_invalid*|*authentication*configuration*)
cause="authentication configuration is invalid"
;;
*workspace_registry_invalid*|*workspace*registry*)
cause="workspace registry startup validation failed"
;;
esac
printf 'Core startup cause: %s (exit code %s).\n' "$cause" "$exit_code" >&2
}
task13_compose_start_logged() {
local label="$1"
shift
task13_compose_files
if task13_bounded "$TASK13_COMMAND_TIMEOUT" "$label" \
task13_compose_invoke "${TASK13_COMPOSE[@]}" "$@" >>"$TASK13_LOG" 2>&1; then
return 0
fi
TASK13_FAILURE_LOGGED=1
printf 'Task 13 command failed: %s\n' "$label" >&2
task13_report_core_startup_failure
return 1
}
task13_write_environment() {
local remote="$1"
{
printf 'THOTH_HTTP_PORT=%s\n' "$TASK13_FRONTEND_PORT"
printf 'THOTH_CORE_HTTP_PORT=0\n'
printf 'MAX_PI_PROCESSES=2\n'
printf 'PI_AUTH_FILE=%s\n' "$TASK13_PI_AUTH"
printf 'THT_SECRETS_FILE=%s\n' "$TASK13_SECRETS"
printf 'THT_INSTALLATION_CONFIG_SOURCE=%s\n' "$TASK13_INSTALLATION"
printf 'THT_AUTH_CONFIG_ROOT=%s\n' "$TASK13_AUTH_ROOT"
printf 'THT_WORKSPACE_GIT_REMOTE=%s\n' "$remote"
printf 'THT_WORKSPACE_GIT_BRANCH=%s\n' "$TASK13_BRANCH"
printf 'THT_LLM_URL=http://%s:9000/v1\n' "$TASK13_LLM_CONTAINER"
} >"$TASK13_ENV_FILE"
chmod 0600 "$TASK13_ENV_FILE"
}
task13_write_session_ca_fixture() {
cat >"$TASK13_SESSION_CA" <<'EOF'
-----BEGIN CERTIFICATE-----
VEFTSzEzLURJU1BPU0FCTEUtU0VTU0lPTi1DQQ==
-----END CERTIFICATE-----
EOF
chmod 0600 "$TASK13_SESSION_CA"
}
# Test-only materialization of the same deterministic boundary adapters covered by the Go
# modelprojection tests. Production lifecycle commands always generate these files themselves.
task13_write_model_projection_fixture() {
python3 - "$TASK13_INSTALLATION" "$TASK13_LLM_CONTAINER" <<'PY'
import hashlib
import json
from pathlib import Path
import sys
installation = Path(sys.argv[1])
provider_host = sys.argv[2]
generated = installation.parent / "generated"
(generated / "pi").mkdir(parents=True, exist_ok=True)
catalog = {
"schemaVersion": 1,
"defaultSession": "local-qwen/task13-smoke",
"embedding": {"id": "ollama/qwen3-embedding:0.6b", "dimensions": 1024},
"models": [{
"id": "local-qwen/task13-smoke",
"provider": "local-qwen",
"model": "task13-smoke",
"label": "Task 13 deterministic smoke",
"upstreamModel": "task13-smoke",
"endpoint": {"baseUrl": f"http://{provider_host}:9000/v1"},
"authentication": {"mode": "none"},
"sessionAdapter": {"mode": "openai_compatible"},
"session": {
"reasoning": False,
"input": ["text"],
"cost": {"input": 0, "output": 0, "cacheRead": 0, "cacheWrite": 0},
"contextWindow": 4096,
"maxTokens": 64,
},
}],
}
models = {
"providers": {"local-qwen": {
"baseUrl": f"http://{provider_host}:9000/v1",
"api": "openai-completions",
"apiKey": "local",
"models": [{
"id": "task13-smoke", "name": "Task 13 deterministic smoke",
"reasoning": False, "input": ["text"],
"cost": {"input": 0, "output": 0, "cacheRead": 0, "cacheWrite": 0},
"contextWindow": 4096, "maxTokens": 64,
}],
}},
}
settings = {"defaultProjectTrust": "always", "enabledModels": ["local-qwen/task13-smoke"]}
serialized = []
for path, value in (
(generated / "catalog.json", catalog),
(generated / "pi/models.json", models),
(generated / "pi/settings.json", settings),
):
raw = json.dumps(value, indent=2, sort_keys=True) + "\n"
path.write_text(raw)
serialized.append(raw.encode())
revision = "sha256:" + hashlib.sha256(b"".join(serialized)).hexdigest()
quote = json.dumps
(generated / "compose.models.yaml").write_text(f"""services:
core:
environment:
THT_MODEL_CATALOG_FILE: /run/thothii-model-catalog/catalog.json
THT_MODEL_CATALOG_REVISION: {quote(revision)}
THT_DEFAULT_SESSION_MODEL: local-qwen/task13-smoke
THT_INTERNAL_EMBEDDING_ID: ollama/qwen3-embedding:0.6b
THT_INTERNAL_EMBEDDING_MODEL: qwen3-embedding:0.6b
THT_INTERNAL_EMBEDDING_DIMENSIONS: '1024'
volumes:
- type: bind
source: {quote(str(generated / 'catalog.json'))}
target: /run/thothii-model-catalog/catalog.json
read_only: true
- type: bind
source: {quote(str(generated / 'pi/models.json'))}
target: /home/thoth/.pi/agent/models.json
read_only: true
- type: bind
source: {quote(str(generated / 'pi/settings.json'))}
target: /home/thoth/.pi/agent/settings.json
read_only: true
workspace-maintenance:
environment:
THT_INTERNAL_EMBEDDING_ID: ollama/qwen3-embedding:0.6b
THT_INTERNAL_EMBEDDING_MODEL: qwen3-embedding:0.6b
THT_INTERNAL_EMBEDDING_DIMENSIONS: '1024'
embedding-model-init:
environment:
OLLAMA_MODEL: qwen3-embedding:0.6b
""")
PY
}
task13_write_fixture_files() {
printf '{}\n' >"$TASK13_PI_AUTH"
printf 'THT_MODEL_API_KEY=%s\n' "$TASK13_SECRET_VALUE" >"$TASK13_SECRETS"
printf '%s' "task13-runtime-password-$TASK13_RUN_ID" >"$TASK13_SESSION_RUNTIME_PASSWORD"
printf '%s' "$TASK13_AUTH_PASSWORD" >"$TASK13_AUTH_PASSWORD_FILE"
task13_write_session_ca_fixture
mkdir -p "$TASK13_AUTH_ROOT"
chmod 0600 "$TASK13_PI_AUTH"
chmod 0700 "$TASK13_AUTH_ROOT"
chmod 0600 "$TASK13_SECRETS" "$TASK13_SESSION_RUNTIME_PASSWORD" "$TASK13_AUTH_PASSWORD_FILE"
cat >"$TASK13_LLM_SERVER" <<'EOF'
import http from "node:http";
const server = http.createServer((request, response) => {
if (request.method === "GET" && request.url === "/health") {
response.writeHead(200, { "content-type": "application/json" });
response.end('{"status":"ok"}');
return;
}
if (request.method === "GET" && request.url === "/v1/models") {
response.writeHead(200, { "content-type": "application/json" });
response.end('{"object":"list","data":[{"id":"task13-smoke","object":"model"}]}');
return;
}
if (request.method !== "POST" || request.url !== "/v1/chat/completions") {
response.writeHead(404, { "content-type": "application/json" });
response.end('{"error":{"message":"not found"}}');
return;
}
let body = "";
request.setEncoding("utf8");
request.on("data", (chunk) => { body += chunk; });
request.on("end", () => {
let stream = true;
try { stream = JSON.parse(body).stream !== false; } catch { /* return the safe fixture */ }
if (!stream) {
response.writeHead(200, { "content-type": "application/json" });
response.end(JSON.stringify({
id: "task13", object: "chat.completion", created: 1, model: "task13-smoke",
choices: [{ index: 0, message: { role: "assistant", content: "OK" }, finish_reason: "stop" }],
}));
return;
}
response.writeHead(200, {
"content-type": "text/event-stream",
"cache-control": "no-cache",
connection: "keep-alive",
});
response.write('data: {"id":"task13","object":"chat.completion.chunk","created":1,"model":"task13-smoke","choices":[{"index":0,"delta":{"role":"assistant","content":"OK"},"finish_reason":null}]}\n\n');
response.write('data: {"id":"task13","object":"chat.completion.chunk","created":1,"model":"task13-smoke","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}\n\n');
response.end("data: [DONE]\n\n");
});
});
server.listen(9000, "0.0.0.0");
EOF
chmod 0644 "$TASK13_LLM_SERVER"
cat >"$TASK13_OVERRIDE" <<EOF
services:
core:
image: $TASK13_CORE_IMAGE
build:
labels:
io.thothii.task13.run: "$TASK13_RUN_ID"
environment:
PI_THINKING: low
THT_WORKSPACE_INSTALLATION_ID: task13-smoke
THT_WORKSPACE_REGISTRY_ROOT: /data/workspace-registry
THT_WS_TASK13_SMOKE_DWH_TRANSPORT: postgres_direct
THT_WS_TASK13_SMOKE_DWH_HOST: dwh.task13.invalid
THT_WS_TASK13_SMOKE_DWH_PORT: "5432"
THT_WS_TASK13_SMOKE_DWH_USER: task13_reader
THT_WS_TASK13_SMOKE_DWH_PASSWORD_FILE: /run/secrets/task13-runtime-password
labels:
io.thothii.task13.run: "$TASK13_RUN_ID"
volumes: !override
- settings:/data/settings
- pi-state:/home/thoth/.pi
- workspace-registry:/data/workspace-registry
- workspace-secrets:/data/workspace-secrets
- sessions:/data/sessions
- auth-runtime:/run/thothii-auth:ro
- auth-state:/data/auth
- application-secrets:/run/secrets:ro
- registry-remote:/fixtures/remote.git:ro
secrets: !reset []
workspace-maintenance:
image: $TASK13_CORE_IMAGE
pull_policy: never
labels:
io.thothii.task13.run: "$TASK13_RUN_ID"
frontend:
image: $TASK13_FRONTEND_IMAGE
build:
labels:
io.thothii.task13.run: "$TASK13_RUN_ID"
labels:
io.thothii.task13.run: "$TASK13_RUN_ID"
qdrant:
# The normal local profile exposes a developer dashboard; the isolated smoke needs no host port.
ports: !reset []
labels:
io.thothii.task13.run: "$TASK13_RUN_ID"
embedding:
labels:
io.thothii.task13.run: "$TASK13_RUN_ID"
embedding-model-init:
labels:
io.thothii.task13.run: "$TASK13_RUN_ID"
networks:
thothii:
labels:
io.thothii.task13.run: "$TASK13_RUN_ID"
volumes:
settings:
labels:
io.thothii.task13.run: "$TASK13_RUN_ID"
pi-state:
labels:
io.thothii.task13.run: "$TASK13_RUN_ID"
workspace-registry:
labels:
io.thothii.task13.run: "$TASK13_RUN_ID"
workspace-secrets:
labels:
io.thothii.task13.run: "$TASK13_RUN_ID"
sessions:
labels:
io.thothii.task13.run: "$TASK13_RUN_ID"
auth-state:
labels:
io.thothii.task13.run: "$TASK13_RUN_ID"
auth-runtime:
labels:
io.thothii.task13.run: "$TASK13_RUN_ID"
application-secrets:
labels:
io.thothii.task13.run: "$TASK13_RUN_ID"
registry-remote:
labels:
io.thothii.task13.run: "$TASK13_RUN_ID"
qdrant-data:
labels:
io.thothii.task13.run: "$TASK13_RUN_ID"
embedding-models:
labels:
io.thothii.task13.run: "$TASK13_RUN_ID"
EOF
chmod 0600 "$TASK13_OVERRIDE"
cat >"$TASK13_INSTALLATION" <<EOF
schemaVersion: 2
profile: local
projectDirectory: "$TASK13_ROOT"
envFile: "$TASK13_ENV_FILE"
modelCatalog:
defaults:
session: local-qwen/task13-smoke
embedding:
id: ollama/qwen3-embedding:0.6b
dimensions: 1024
providers:
local-qwen:
endpoint:
baseUrl: http://$TASK13_LLM_CONTAINER:9000/v1
authentication:
mode: none
session:
mode: openai_compatible
models:
task13-smoke:
label: Task 13 deterministic smoke
session:
reasoning: false
input: [text]
cost: {input: 0, output: 0, cacheRead: 0, cacheWrite: 0}
contextWindow: 4096
maxTokens: 64
authentication:
configDirectory: "$TASK13_AUTH_ROOT"
overrides:
- "$TASK13_OVERRIDE"
EOF
chmod 0600 "$TASK13_INSTALLATION"
task13_write_model_projection_fixture
}
task13_write_server_fixture_files() {
local data_root pi_root registry_root remote_path workspace_path
TASK13_OIDC_SERVER="${TASK13_OIDC_SERVER:-$TASK13_TMP/fake-oidc.mjs}"
TASK13_OIDC_CERT="${TASK13_OIDC_CERT:-$TASK13_TMP/fake-oidc-cert.pem}"
TASK13_OIDC_KEY="${TASK13_OIDC_KEY:-$TASK13_TMP/fake-oidc-key.pem}"
printf '{}\n' >"$TASK13_PI_AUTH"
printf 'THT_MODEL_API_KEY=%s\nTHT_OIDC_CLIENT_SECRET=%s\nTHT_AUTHENTIK_API_TOKEN=%s\n' \
"$TASK13_SECRET_VALUE" "$TASK13_OIDC_CLIENT_SECRET" "$TASK13_AUTHENTIK_API_TOKEN" >"$TASK13_SECRETS"
printf '%s' "$TASK13_SESSION_PASSWORD" >"$TASK13_SESSION_RUNTIME_PASSWORD"
printf '%s' "$TASK13_SESSION_MIGRATOR_PASSWORD" >"$TASK13_SESSION_MIGRATOR_PASSWORD_FILE"
task13_write_session_ca_fixture
chmod 0600 "$TASK13_PI_AUTH"
chmod 0600 "$TASK13_SECRETS" "$TASK13_SESSION_RUNTIME_PASSWORD" \
"$TASK13_SESSION_MIGRATOR_PASSWORD_FILE"
task13_run_logged "create scoped fake OIDC certificate" openssl req -x509 -newkey rsa:2048 \
-sha256 -nodes -days 1 -subj '/CN=task13-fake-oidc' \
-addext 'subjectAltName=DNS:task13-fake-oidc' \
-keyout "$TASK13_OIDC_KEY" -out "$TASK13_OIDC_CERT"
chmod 0600 "$TASK13_OIDC_KEY"
chmod 0644 "$TASK13_OIDC_CERT"
cat >"$TASK13_OIDC_SERVER" <<'EOF'
import { generateKeyPairSync } from "node:crypto";
import { readFileSync } from "node:fs";
import https from "node:https";
const origin = "https://task13-fake-oidc:9443";
const issuer = `${origin}/application/o/task13/`;
const expectedToken = process.env.TASK13_AUTHENTIK_API_TOKEN;
const { publicKey } = generateKeyPairSync("rsa", { modulusLength: 2048 });
const jwk = { ...publicKey.export({ format: "jwk" }), kid: "task13", use: "sig", alg: "RS256" };
const send = (response, status, body) => {
const payload = JSON.stringify(body);
response.writeHead(status, { "content-type": "application/json", "content-length": Buffer.byteLength(payload) });
response.end(payload);
};
const server = https.createServer({
cert: readFileSync("/fixtures/oidc-cert.pem"),
key: readFileSync("/fixtures/oidc-key.pem"),
}, (request, response) => {
const target = new URL(request.url ?? "/", origin);
if (target.pathname === "/health") return send(response, 200, { status: "ok" });
if (target.pathname.includes(".well-known/openid-configuration")) {
return send(response, 200, {
issuer,
authorization_endpoint: `${origin}/authorize`,
token_endpoint: `${origin}/token`,
jwks_uri: `${origin}/jwks`,
response_types_supported: ["code"],
subject_types_supported: ["public"],
id_token_signing_alg_values_supported: ["RS256"],
});
}
if (target.pathname === "/jwks") return send(response, 200, { keys: [jwk] });
if (target.pathname === "/api/v3/core/groups/") {
if (request.headers.authorization !== `Bearer ${expectedToken}`) return send(response, 401, { detail: "unauthorized" });
const name = target.searchParams.get("name") ?? "";
console.log(`group:${name}`);
const configured = name === "task13-users" || name === "task13-admins";
return send(response, 200, {
pagination: { next: null },
results: configured ? [{ name }, { name: "task13-unrelated" }] : [{ name: "task13-unrelated" }],
});
}
return send(response, 404, { error: "not_found" });
});
server.listen(9443, "0.0.0.0");
EOF
chmod 0644 "$TASK13_OIDC_SERVER"
cat >"$TASK13_SERVER_WORKSPACE_CONFIG" <<'EOF'
language: en
session_storage:
type: postgres_direct
connection:
host: ${THT_SESSION_DB_HOST}
port: ${THT_SESSION_DB_PORT}
database: ${THT_SESSION_DB_NAME}
schema: thoth_sessions
user: ${THT_SESSION_RUNTIME_USER}
password_file: ${THT_SESSION_RUNTIME_PASSWORD_FILE}
sslmode: ${THT_SESSION_DB_SSLMODE}
sslrootcert: ${THT_SESSION_DB_SSLROOTCERT}
roots:
artifacts: artifacts
indexes: indexes
sessions: sessions
EOF
chmod 0644 "$TASK13_SERVER_WORKSPACE_CONFIG"
mkdir -p "$TASK13_SERVER_DATA" "$TASK13_SERVER_PI_STATE" "$TASK13_SERVER_REGISTRY"
"$TASK13_ROOT/scripts/prepare-server-pi-state.sh" \
"$TASK13_SERVER_PI_STATE" "$(id -u)" "$(id -g)" >>"$TASK13_LOG"
chmod 0777 "$TASK13_SERVER_DATA" "$TASK13_SERVER_PI_STATE" \
"$TASK13_SERVER_PI_STATE/agent" "$TASK13_SERVER_REGISTRY"
data_root="$TASK13_SERVER_DATA"
pi_root="$TASK13_SERVER_PI_STATE"
registry_root="$TASK13_SERVER_REGISTRY"
remote_path="$TASK13_REMOTE"
workspace_path="$TASK13_SERVER_WORKSPACE_CONFIG"
cat >"$TASK13_OVERRIDE" <<EOF
services:
core:
image: $TASK13_CORE_IMAGE
build:
labels:
io.thothii.task13.run: "$TASK13_RUN_ID"
environment:
NODE_EXTRA_CA_CERTS: /fixtures/task13-oidc-ca.pem
THT_WS_TASK13_SMOKE_DWH_TRANSPORT: postgres_direct
THT_WS_TASK13_SMOKE_DWH_HOST: dwh.task13.invalid
THT_WS_TASK13_SMOKE_DWH_PORT: "5432"
THT_WS_TASK13_SMOKE_DWH_USER: task13_reader
THT_WS_TASK13_SMOKE_DWH_PASSWORD_FILE: /run/secrets/task13-runtime-password
labels:
io.thothii.task13.run: "$TASK13_RUN_ID"
volumes:
- $remote_path:/fixtures/remote.git:ro
- $TASK13_SESSION_RUNTIME_PASSWORD:/run/secrets/task13-runtime-password:ro
- $TASK13_OIDC_CERT:/fixtures/task13-oidc-ca.pem:ro
workspace-maintenance:
image: $TASK13_CORE_IMAGE
pull_policy: never
labels:
io.thothii.task13.run: "$TASK13_RUN_ID"
frontend:
image: $TASK13_FRONTEND_IMAGE
build:
labels:
io.thothii.task13.run: "$TASK13_RUN_ID"
labels:
io.thothii.task13.run: "$TASK13_RUN_ID"
qdrant:
labels:
io.thothii.task13.run: "$TASK13_RUN_ID"
embedding:
labels:
io.thothii.task13.run: "$TASK13_RUN_ID"
embedding-model-init:
labels:
io.thothii.task13.run: "$TASK13_RUN_ID"
session-migrate:
image: $TASK13_CORE_IMAGE
networks:
thothii:
labels:
io.thothii.task13.run: "$TASK13_RUN_ID"
volumes:
qdrant-data:
labels:
io.thothii.task13.run: "$TASK13_RUN_ID"
embedding-models:
labels:
io.thothii.task13.run: "$TASK13_RUN_ID"
EOF
chmod 0600 "$TASK13_OVERRIDE"
{
printf 'THOTH_HTTP_PORT=0\n'
printf 'THOTH_SERVER_BIND=127.0.0.1\n'
printf 'MAX_PI_PROCESSES=2\n'
printf 'PI_AUTH_FILE=%s\n' "$TASK13_PI_AUTH"
printf 'THT_SECRETS_FILE=%s\n' "$TASK13_SECRETS"
printf 'THT_INSTALLATION_CONFIG_SOURCE=%s\n' "$TASK13_INSTALLATION"
printf 'THT_AUTH_CONFIG_ROOT=%s\n' "$TASK13_AUTH_ROOT"
printf 'THT_AUTH_RUNTIME_ROOT=%s\n' "$TASK13_AUTH_RUNTIME_ROOT"
printf 'THT_WORKSPACE_GIT_REMOTE=/fixtures/remote.git\n'
printf 'THT_WORKSPACE_GIT_BRANCH=%s\n' "$TASK13_BRANCH"
printf 'THT_DATA_ROOT=%s\n' "$data_root"
printf 'THT_PI_STATE_ROOT=%s\n' "$pi_root"
printf 'THT_WORKSPACE_REGISTRY_ROOT=%s\n' "$registry_root"
printf 'THT_SERVER_WORKSPACE_CONFIG=%s\n' "$workspace_path"
printf 'THT_SESSION_DB_HOST=task13-session.invalid\n'
printf 'THT_SESSION_DB_PORT=5432\n'
printf 'THT_SESSION_DB_NAME=task13\n'
printf 'THT_SESSION_RUNTIME_USER=task13_runtime\n'
printf 'THT_SESSION_MIGRATOR_USER=task13_migrator\n'
printf 'THT_SESSION_DB_SSLMODE=verify-full\n'
printf 'THT_SESSION_RUNTIME_PASSWORD_SOURCE=%s\n' "$TASK13_SESSION_RUNTIME_PASSWORD"
printf 'THT_SESSION_MIGRATOR_PASSWORD_SOURCE=%s\n' "$TASK13_SESSION_MIGRATOR_PASSWORD_FILE"
printf 'THT_SESSION_CA_SOURCE=%s\n' "$TASK13_SESSION_CA"
printf 'THT_LLM_URL=https://llm.task13.invalid/v1\n'
} >"$TASK13_ENV_FILE"
chmod 0600 "$TASK13_ENV_FILE"
cat >"$TASK13_INSTALLATION" <<EOF
schemaVersion: 2
profile: server
projectDirectory: "$TASK13_ROOT"
envFile: "$TASK13_ENV_FILE"
modelCatalog:
defaults:
session: local-qwen/task13-smoke
embedding:
id: ollama/qwen3-embedding:0.6b
dimensions: 1024
providers:
local-qwen:
endpoint:
baseUrl: http://$TASK13_LLM_CONTAINER:9000/v1
authentication:
mode: none
session:
mode: openai_compatible
models:
task13-smoke:
label: Task 13 deterministic smoke
session:
reasoning: false
input: [text]
cost: {input: 0, output: 0, cacheRead: 0, cacheWrite: 0}
contextWindow: 4096
maxTokens: 64
authentication:
configDirectory: "$TASK13_AUTH_ROOT"
runtimeProjection:
directory: "$TASK13_AUTH_RUNTIME_ROOT"
uid: 10001
gid: 10001
overrides:
- "$TASK13_ROOT/deploy/compose.session-server.yaml.example"
- "$TASK13_OVERRIDE"
EOF
chmod 0600 "$TASK13_INSTALLATION"
task13_write_model_projection_fixture
}
task13_workspace_metadata() {
local fixture="$1"
awk '
/^workspace:[[:space:]]*$/ { in_workspace = 1; next }
in_workspace && /^[^[:space:]]/ { in_workspace = 0 }
in_workspace && /^ id:[[:space:]]*/ {
sub(/^ id:[[:space:]]*/, "")
id = $0
next
}
in_workspace && /^ name:[[:space:]]*/ {
sub(/^ name:[[:space:]]*/, "")
name = $0
next
}
in_workspace && /^ description:[[:space:]]*/ {
sub(/^ description:[[:space:]]*/, "")
description = $0
next
}
END {
if (id == "" || name == "") exit 1
print id
print name
if (description != "") print description
}
' "$fixture"
}
task13_write_catalog() {
local catalog_path="$1" id="$2" name="$3" description="${4:-}"
{
printf 'schema_version: 1\n'
printf 'workspaces:\n'
printf ' - id: %s\n' "$id"
printf ' name: %s\n' "$name"
if [[ -n "$description" ]]; then
printf ' description: %s\n' "$description"
fi
} >"$catalog_path"
}
task13_replace_once() {
local target="$1" old="$2" new="$3"
python3 - "$target" "$old" "$new" <<'PY'
from pathlib import Path
import sys
path = Path(sys.argv[1])
old = sys.argv[2]
new = sys.argv[3]
text = path.read_text()
count = text.count(old)
if count != 1:
raise SystemExit(f"expected exactly one occurrence of {old!r} in {path}, found {count}")
path.write_text(text.replace(old, new, 1))
PY
}
task13_commit_registry_change() {
local message="$1"
task13_run_logged "$message" git -C "$TASK13_SEED" add -A
task13_run_logged "$message" git -C "$TASK13_SEED" -c user.name='Task 13 Smoke' -c user.email='task13-smoke@example.invalid' commit -m "$message"
task13_run_logged "$message" git -C "$TASK13_SEED" push "$TASK13_REMOTE" "HEAD:$TASK13_BRANCH"
if [[ -n "${TASK13_REGISTRY_RUNTIME_VOLUME:-}" ]] \
&& docker volume inspect "$TASK13_REGISTRY_RUNTIME_VOLUME" >/dev/null 2>&1; then
task13_prepare_registry_remote
fi
}
task13_seed_registry() {
local fixture metadata
fixture="$TASK13_ROOT/scripts/fixtures/workspace-registry-task13.yaml"
mapfile -t metadata < <(task13_workspace_metadata "$fixture")
TASK13_WORKSPACE_ID="${metadata[0]}"
TASK13_WORKSPACE_NAME="${metadata[1]}"
TASK13_WORKSPACE_DESCRIPTION="${metadata[2]-}"
mkdir -p "$TASK13_SEED"
task13_run_logged "initialize bare workspace registry" git init --bare --initial-branch=main "$TASK13_REMOTE"
task13_run_logged "initialize workspace seed" git -C "$TASK13_SEED" init --initial-branch=main
mkdir -p "$TASK13_SEED/$TASK13_WORKSPACE_ID"
cp "$fixture" "$TASK13_SEED/$TASK13_WORKSPACE_ID/workspace.yaml"
task13_write_catalog "$TASK13_SEED/thoth-workspaces.yaml" "$TASK13_WORKSPACE_ID" "$TASK13_WORKSPACE_NAME" "$TASK13_WORKSPACE_DESCRIPTION"
if grep -Fq 'type: filesystem' "$fixture"; then
mkdir -p "$TASK13_SEED/$TASK13_WORKSPACE_ID/evidence"
printf 'guide v1\n' >"$TASK13_SEED/$TASK13_WORKSPACE_ID/evidence/guide.md"
fi
task13_commit_registry_change 'Seed Task 13 workspace registry'
}
task13_build_tht() {
local os arch
mkdir -p "$TASK13_THT_DIR"
task13_run_logged "build tht cross-platform binaries" env \
THT_THT_OUTPUT_DIRECTORY="$TASK13_THT_DIR" \
bash "$TASK13_ROOT/scripts/build-tht.sh"
os="$(uname -s)"
arch="$(uname -m)"
case "$os/$arch" in
Darwin/x86_64) TASK13_THT="$TASK13_THT_DIR/tht-darwin-amd64" ;;
Darwin/arm64) TASK13_THT="$TASK13_THT_DIR/tht-darwin-arm64" ;;
Linux/x86_64|Linux/amd64) TASK13_THT="$TASK13_THT_DIR/tht-linux-amd64" ;;
Linux/aarch64|Linux/arm64) TASK13_THT="$TASK13_THT_DIR/tht-linux-arm64" ;;
*) task13_fail "unsupported smoke host: $os/$arch" ;;
esac
chmod 0700 "$TASK13_THT"
task13_run_logged "invoke host tht" "$TASK13_THT" --help
}
task13_assert_rendered_contract() {
local services rendered
services="$(task13_compose config --services | sort)"
[[ "$services" == $'catalog-db\ncore\nembedding\nembedding-model-init\nfrontend\nqdrant' ]] \
|| task13_fail "rendered stack is not the mandatory internal semantic topology"
rendered="$TASK13_TMP/rendered-compose.yaml"
task13_compose config >"$rendered"
if grep -Eqi 'docker\.sock|/var/run/docker' "$rendered"; then
task13_fail "rendered Compose exposes a Docker daemon endpoint"
fi
if grep -Fq "$TASK13_SECRET_VALUE" "$rendered"; then
task13_fail "rendered Compose exposed the fixture secret"
fi
for endpoint in THT_DWH_REST_URL THT_LLM_URL \
THT_INTERNAL_QDRANT_URL THT_INTERNAL_EMBEDDING_URL \
THT_INTERNAL_EMBEDDING_MODEL THT_INTERNAL_EMBEDDING_DIMENSIONS; do
grep -Fq "$endpoint" "$rendered" || task13_fail "rendered Compose lacks $endpoint"
done
if grep -Eq 'THT_VEC_REST_URL|THT_VEC_WRITE_REST_URL|THT_OLLAMA_URL' "$rendered"; then
task13_fail "rendered Compose still exposes retired external semantic bindings"
fi
grep -Fq '/run/thothii-auth' "$rendered" || task13_fail "rendered Compose lacks the read-only auth configuration mount"
grep -Fq '/data/auth' "$rendered" || task13_fail "rendered Compose lacks authentication state storage"
}
task13_configure_local_authentication() {
task13_run_logged "configure local authentication" "$TASK13_THT" --installation "$TASK13_INSTALLATION" auth configure \
--mode local --public-url "http://127.0.0.1:$TASK13_FRONTEND_PORT" \
--admin-user "$TASK13_AUTH_ADMIN" --admin-display-name "Task 13 Administrator" \
--password-file "$TASK13_AUTH_PASSWORD_FILE"
}
task13_configure_server_oidc_authentication() {
task13_run_logged "configure fake server OIDC authentication" task13_server_tht auth configure \
--mode oidc --public-url "https://task13.example.invalid" \
--issuer "https://task13-fake-oidc:9443/application/o/task13/" --client-id task13-smoke-client \
--authentik-base-url "https://task13-fake-oidc:9443" --user-group task13-users --admin-group task13-admins
}
task13_prepare_server_auth_roots() {
[[ "${TASK13_PROFILE:-}" == server \
&& "$TASK13_AUTH_ROOT" == "$TASK13_TMP/auth" \
&& ! -L "$TASK13_AUTH_ROOT" \
&& ( ! -e "$TASK13_AUTH_ROOT" || -d "$TASK13_AUTH_ROOT" ) \
&& "$TASK13_AUTH_RUNTIME_ROOT" == "$TASK13_TMP/auth-runtime" \
&& ! -L "$TASK13_AUTH_RUNTIME_ROOT" \
&& ( ! -e "$TASK13_AUTH_RUNTIME_ROOT" || -d "$TASK13_AUTH_RUNTIME_ROOT" ) ]] \
|| task13_fail "refusing to prepare unexpected server authentication roots"
task13_run_logged "prepare server authentication canonical root" sudo -n -- \
install -d -o 0 -g 0 -m 0700 -- "$TASK13_AUTH_ROOT"
task13_run_logged "prepare server authentication runtime root" sudo -n -- \
install -d -o 10001 -g 10001 -m 0700 -- "$TASK13_AUTH_RUNTIME_ROOT"
}
task13_prepare_server_secret_sources() {
local path
[[ "${TASK13_PROFILE:-}" == server && -n "${TASK13_TMP:-}" ]] \
|| task13_fail "refusing to prepare server secret sources outside the server fixture"
for path in \
"$TASK13_SECRETS" \
"$TASK13_PI_AUTH" \
"$TASK13_SESSION_RUNTIME_PASSWORD" \
"$TASK13_SESSION_MIGRATOR_PASSWORD_FILE" \
"$TASK13_SESSION_CA"; do
[[ "$path" == "$TASK13_TMP/"* && -f "$path" && ! -L "$path" ]] \
|| task13_fail "refusing to prepare an unexpected server secret source"
done
task13_run_logged "assign server secret sources to the container UID" sudo -n -- \
chown 10001:10001 -- "$TASK13_SECRETS" "$TASK13_PI_AUTH" \
"$TASK13_SESSION_RUNTIME_PASSWORD" "$TASK13_SESSION_MIGRATOR_PASSWORD_FILE" \
"$TASK13_SESSION_CA"
task13_run_logged "protect server secret sources" sudo -n -- chmod 0600 -- \
"$TASK13_SECRETS" "$TASK13_PI_AUTH" "$TASK13_SESSION_RUNTIME_PASSWORD" \
"$TASK13_SESSION_MIGRATOR_PASSWORD_FILE" "$TASK13_SESSION_CA"
}
task13_server_tht() {
sudo -n -- "$TASK13_THT" --installation "$TASK13_INSTALLATION" "$@"
}
task13_server_checkpoint_leftover() {
[[ "${TASK13_PROFILE:-local}" == server \
&& -n "${TASK13_CONTROL_DIR:-}" \
&& "$TASK13_CONTROL_DIR" == "$TASK13_ROOT/.tht/$TASK13_PROJECT" \
&& "$TASK13_PROJECT" =~ ^thothii-[0-9a-f]{12}$ ]] \
|| task13_fail "refusing to inspect an unexpected server control path"
sudo -n -- find "$TASK13_CONTROL_DIR" -maxdepth 1 -name 'restore-checkpoint-*.zip' -print -quit
}
task13_prepare_local_auth_runtime() {
local owner_label
owner_label="$(docker volume inspect --format '{{ index .Labels "io.thothii.task13.run" }}' \
"$TASK13_AUTH_RUNTIME_VOLUME")"
[[ "$owner_label" == "$TASK13_RUN_ID" ]] \
|| task13_fail "local authentication runtime volume lacks the Task 13 run label"
task13_run_logged "project local authentication for the core runtime" docker run --rm \
--name "$TASK13_AUTH_PROJECTION_CONTAINER" \
--label "io.thothii.task13.run=$TASK13_RUN_ID" \
--user 0:0 \
--entrypoint sh \
--volume "$TASK13_AUTH_ROOT:/source:ro" \
--volume "$TASK13_AUTH_RUNTIME_VOLUME:/target" \
"$TASK13_CORE_IMAGE" -ceu '
test -f /source/auth.yaml && test ! -L /source/auth.yaml
test -f /source/users.yaml && test ! -L /source/users.yaml
test -z "$(find /target -mindepth 1 -maxdepth 1 -print -quit)"
cp /source/auth.yaml /source/users.yaml /target/
chown 10001:10001 /target /target/auth.yaml /target/users.yaml
chmod 0700 /target
chmod 0600 /target/auth.yaml /target/users.yaml
test "$(stat -c "%u:%g:%a" /target)" = 10001:10001:700
test "$(stat -c "%u:%g:%a" /target/auth.yaml)" = 10001:10001:600
test "$(stat -c "%u:%g:%a" /target/users.yaml)" = 10001:10001:600
'
}
task13_prepare_local_pi_runtime() {
local owner_label
owner_label="$(docker volume inspect --format '{{ index .Labels "io.thothii.task13.run" }}' \
"$TASK13_PI_RUNTIME_VOLUME")"
[[ "$owner_label" == "$TASK13_RUN_ID" ]] \
|| task13_fail "local Pi runtime volume lacks the Task 13 run label"
task13_run_logged "project local Pi configuration for the core runtime" docker run --rm \
--name "$TASK13_PI_PROJECTION_CONTAINER" \
--label "io.thothii.task13.run=$TASK13_RUN_ID" \
--user 0:0 \
--entrypoint sh \
--volume "$TASK13_PI_AUTH:/source/auth.json:ro" \
--volume "$TASK13_PI_RUNTIME_VOLUME:/target" \
"$TASK13_CORE_IMAGE" -ceu '
test -f /source/auth.json && test ! -L /source/auth.json
test -d /target/agent && test ! -L /target/agent
test -z "$(find /target -mindepth 1 -maxdepth 1 ! -name agent -print -quit)"
test -z "$(find /target/agent -mindepth 1 -maxdepth 1 -print -quit)"
cp /source/auth.json /target/agent/
chown -R 10001:10001 /target
chmod 0700 /target /target/agent
chmod 0600 /target/agent/auth.json
test "$(stat -c "%u:%g:%a" /target/agent/auth.json)" = 10001:10001:600
'
}
task13_prepare_local_application_secrets() {
local owner_label
owner_label="$(docker volume inspect --format '{{ index .Labels "io.thothii.task13.run" }}' \
"$TASK13_APPLICATION_SECRETS_VOLUME")"
[[ "$owner_label" == "$TASK13_RUN_ID" ]] \
|| task13_fail "application-secret runtime volume lacks the Task 13 run label"
task13_run_logged "project local application secrets for the core runtime" docker run --rm \
--name "$TASK13_APPLICATION_SECRETS_PROJECTION_CONTAINER" \
--label "io.thothii.task13.run=$TASK13_RUN_ID" \
--user 0:0 \
--entrypoint sh \
--volume "$TASK13_SECRETS:/source/thothii.secrets:ro" \
--volume "$TASK13_SESSION_RUNTIME_PASSWORD:/source/task13-runtime-password:ro" \
--volume "$TASK13_SESSION_CA:/source/session_ca.pem:ro" \
--volume "$TASK13_APPLICATION_SECRETS_VOLUME:/target" \
"$TASK13_CORE_IMAGE" -ceu '
test -f /source/thothii.secrets && test ! -L /source/thothii.secrets
test -f /source/task13-runtime-password && test ! -L /source/task13-runtime-password
test -f /source/session_ca.pem && test ! -L /source/session_ca.pem
test -z "$(find /target -mindepth 1 -maxdepth 1 -print -quit)"
cp /source/thothii.secrets /source/task13-runtime-password /source/session_ca.pem /target/
cp /source/task13-runtime-password /target/session_runtime_password
chown 0:0 /target
chown 10001:10001 /target/thothii.secrets /target/task13-runtime-password /target/session_runtime_password /target/session_ca.pem
chmod 0755 /target
chmod 0600 /target/thothii.secrets /target/task13-runtime-password /target/session_runtime_password /target/session_ca.pem
test "$(stat -c "%u:%g:%a" /target)" = 0:0:755
test "$(stat -c "%u:%g:%a" /target/thothii.secrets)" = 10001:10001:600
test "$(stat -c "%u:%g:%a" /target/task13-runtime-password)" = 10001:10001:600
test "$(stat -c "%u:%g:%a" /target/session_runtime_password)" = 10001:10001:600
test "$(stat -c "%u:%g:%a" /target/session_ca.pem)" = 10001:10001:600
'
}
task13_prepare_registry_remote() {
local owner_label
owner_label="$(docker volume inspect --format '{{ index .Labels "io.thothii.task13.run" }}' \
"$TASK13_REGISTRY_RUNTIME_VOLUME")"
[[ "$owner_label" == "$TASK13_RUN_ID" ]] \
|| task13_fail "registry fixture runtime volume lacks the Task 13 run label"
task13_run_logged "project Git fixture for the core runtime" docker run --rm \
--name "$TASK13_REGISTRY_PROJECTION_CONTAINER" \
--label "io.thothii.task13.run=$TASK13_RUN_ID" \
--user 0:0 \
--entrypoint sh \
--volume "$TASK13_REMOTE:/source:ro" \
--volume "$TASK13_REGISTRY_RUNTIME_VOLUME:/target" \
"$TASK13_CORE_IMAGE" -ceu '
test -f /source/HEAD && test -d /source/objects && test -d /source/refs
cp -a /source/. /target/
chown -R 10001:10001 /target
chmod -R u=rwX,go= /target
test "$(stat -c "%u:%g:%a" /target)" = 10001:10001:700
test "$(stat -c "%u:%g" /target/HEAD)" = 10001:10001
'
}
task13_start_stack() {
printf '== Build and start isolated local Compose distribution ==\n'
task13_assert_rendered_contract
task13_compose_logged "build local Compose images" build --pull
task13_compose_logged "create local core authentication runtime" create core
task13_prepare_local_auth_runtime
task13_prepare_local_pi_runtime
task13_prepare_local_application_secrets
task13_prepare_registry_remote
task13_compose_start_logged "start local Compose distribution" up --detach --wait --wait-timeout 120
TASK13_NETWORK="$(docker network ls \
--filter "label=com.docker.compose.project=$TASK13_PROJECT" \
--filter 'label=com.docker.compose.network=thothii' --format '{{.Name}}')"
[[ -n "$TASK13_NETWORK" && "$TASK13_NETWORK" != *$'\n'* ]] || task13_fail "isolated Compose network was not resolved"
task13_run_logged "start deterministic local LLM fixture" docker run --detach \
--name "$TASK13_LLM_CONTAINER" \
--label "io.thothii.task13.run=$TASK13_RUN_ID" \
--network "$TASK13_NETWORK" \
--entrypoint node \
--volume "$TASK13_LLM_SERVER:/fixtures/fake-llm.mjs:ro" \
"$TASK13_CORE_IMAGE" /fixtures/fake-llm.mjs
for _attempt in $(seq 1 30); do
if docker exec "$TASK13_LLM_CONTAINER" node -e \
"fetch('http://127.0.0.1:9000/health',{signal:AbortSignal.timeout(3000)}).then(r=>{if(!r.ok)process.exit(1)}).catch(()=>process.exit(1))" \
>>"$TASK13_LOG" 2>&1; then
return 0
fi
sleep 1
done
task13_log_failure "deterministic local LLM fixture readiness"
}
task13_start_server_stack() {
printf '== Build and start isolated Linux server profile ==\n'
task13_assert_rendered_contract
task13_compose_logged "build server Compose images" build --pull core frontend
task13_compose_logged "create server Compose resources" create core frontend
TASK13_NETWORK="$(docker network ls \
--filter "label=com.docker.compose.project=$TASK13_PROJECT" \
--filter 'label=com.docker.compose.network=thothii' --format '{{.Name}}')"
[[ -n "$TASK13_NETWORK" && "$TASK13_NETWORK" != *$'\n'* ]] \
|| task13_fail "isolated server Compose network was not resolved"
task13_run_logged "start scoped fake OIDC provider" docker run --detach \
--name "$TASK13_OIDC_CONTAINER" \
--label "io.thothii.task13.run=$TASK13_RUN_ID" \
--network "$TASK13_NETWORK" --network-alias task13-fake-oidc \
--user "$(id -u):$(id -g)" \
--env "TASK13_AUTHENTIK_API_TOKEN=$TASK13_AUTHENTIK_API_TOKEN" \
--env NODE_EXTRA_CA_CERTS=/fixtures/oidc-cert.pem \
--entrypoint node \
--volume "$TASK13_OIDC_SERVER:/fixtures/fake-oidc.mjs:ro" \
--volume "$TASK13_OIDC_CERT:/fixtures/oidc-cert.pem:ro" \
--volume "$TASK13_OIDC_KEY:/fixtures/oidc-key.pem:ro" \
"$TASK13_CORE_IMAGE" /fixtures/fake-oidc.mjs
for _attempt in $(seq 1 30); do
if docker exec "$TASK13_OIDC_CONTAINER" node -e \
"fetch('https://task13-fake-oidc:9443/health',{signal:AbortSignal.timeout(3000)}).then(r=>{if(!r.ok)process.exit(1)}).catch(()=>process.exit(1))" \
>>"$TASK13_LOG" 2>&1; then
break
fi
sleep 1
done
docker exec "$TASK13_OIDC_CONTAINER" node -e \
"fetch('https://task13-fake-oidc:9443/health',{signal:AbortSignal.timeout(3000)}).then(r=>{if(!r.ok)process.exit(1)}).catch(()=>process.exit(1))" \
>>"$TASK13_LOG" 2>&1 || task13_log_failure "scoped fake OIDC provider readiness"
task13_compose_start_logged "start server Compose distribution" \
up --detach --wait --wait-timeout 120 core frontend
}
task13_frontend_address() {
task13_compose port frontend 8080 | awk 'NR == 1 {print $0}'
}
task13_core_id() {
task13_compose ps -q core
}
task13_assert_maintenance_auth_isolation() {
local rendered
rendered="$TASK13_TMP/maintenance-auth-isolation.json"
if ! task13_compose --profile workspace-maintenance config --format json >"$rendered" 2>>"$TASK13_LOG"; then
task13_log_failure "render workspace maintenance mount isolation"
fi
if ! node - "$rendered" <<'NODE'
const config = JSON.parse(require("node:fs").readFileSync(process.argv[2], "utf8"));
const maintenance = config.services?.["workspace-maintenance"];
if (!maintenance || !Array.isArray(maintenance.volumes)) process.exit(1);
if (maintenance.volumes.some((mount) => mount?.target === "/run/thothii-auth" || mount?.target === "/data/auth")) {
process.exit(1);
}
NODE
then
task13_log_failure "workspace maintenance authentication mount isolation"
fi
task13_compose_logged "seed core authentication isolation sentinel" exec -T core sh -ceu \
': > /data/auth/task13-maintenance-isolation-sentinel'
task13_compose_logged "workspace maintenance auth isolation" \
--profile workspace-maintenance run --rm --no-deps --entrypoint sh workspace-maintenance -ceu \
'test ! -e /run/thothii-auth
test -d /data/auth
test ! -e /data/auth/task13-maintenance-isolation-sentinel'
task13_compose_logged "verify core authentication isolation sentinel" exec -T core sh -ceu \
'test -f /data/auth/task13-maintenance-isolation-sentinel'
task13_compose_logged "remove core authentication isolation sentinel" exec -T core sh -ceu \
'rm -f /data/auth/task13-maintenance-isolation-sentinel'
}
task13_assert_local_auth_lifecycle() {
local frontend cookie_jar login_body me csrf unauthenticated
frontend="$(task13_frontend_address)"
cookie_jar="$TASK13_TMP/local-auth.cookies"
login_body="$TASK13_TMP/local-auth-login.json"
printf '{"username":"%s","password":"%s","remember":true}' \
"$TASK13_AUTH_ADMIN" "$TASK13_AUTH_PASSWORD" >"$login_body"
chmod 0600 "$cookie_jar" "$login_body" 2>/dev/null || chmod 0600 "$login_body"
task13_run_logged "local auth configuration" curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" \
--max-time "$TASK13_CURL_MAX_TIME" --fail --silent --show-error "http://$frontend/api/auth/config"
task13_run_logged "local auth login" curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" \
--max-time "$TASK13_CURL_MAX_TIME" --fail --silent --show-error --cookie-jar "$cookie_jar" \
-H "Origin: http://$frontend" -H 'content-type: application/json' --data-binary "@$login_body" \
"http://$frontend/api/auth/local/login"
me="$(curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" --max-time "$TASK13_CURL_MAX_TIME" \
--fail --silent --show-error --cookie "$cookie_jar" "http://$frontend/api/me")"
node -e 'const value=JSON.parse(process.argv[1]); if(value.issuer!=="local"||value.session?.remembered!==true||typeof value.csrfToken!=="string") process.exit(1)' "$me" \
|| task13_fail "local login did not create a remembered authenticated session"
csrf="$(node -e 'process.stdout.write(JSON.parse(process.argv[1]).csrfToken)' "$me")"
task13_compose_logged "remembered local auth core restart" up --detach --force-recreate --wait --wait-timeout 120 core
me="$(curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" --max-time "$TASK13_CURL_MAX_TIME" \
--fail --silent --show-error --cookie "$cookie_jar" "http://$frontend/api/me")"
node -e 'const value=JSON.parse(process.argv[1]); if(value.session?.remembered!==true) process.exit(1)' "$me" \
|| task13_fail "remembered local session did not survive a core restart"
csrf="$(node -e 'process.stdout.write(JSON.parse(process.argv[1]).csrfToken)' "$me")"
task13_run_logged "local auth Pi management" curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" \
--max-time 45 --fail --silent --show-error --cookie "$cookie_jar" \
-H "Origin: http://$frontend" -H "x-thothii-csrf: $csrf" -X POST "http://$frontend/api/pi-management/test"
task13_run_logged "local auth logout" curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" \
--max-time "$TASK13_CURL_MAX_TIME" --fail --silent --show-error --cookie "$cookie_jar" \
-H "Origin: http://$frontend" -H "x-thothii-csrf: $csrf" -X POST "http://$frontend/api/auth/logout"
unauthenticated="$(curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" --max-time "$TASK13_CURL_MAX_TIME" \
--silent --output /dev/null --write-out '%{http_code}' --cookie "$cookie_jar" "http://$frontend/api/me")"
[[ "$unauthenticated" == 401 ]] || task13_fail "local logout did not revoke the remembered session"
}
task13_create_admin_session() {
local frontend login_body me
frontend="$(task13_frontend_address)"
TASK13_ADMIN_COOKIE="$TASK13_TMP/operations-admin.cookies"
login_body="$TASK13_TMP/operations-admin-login.json"
printf '{"username":"%s","password":"%s","remember":true}' \
"$TASK13_AUTH_ADMIN" "$TASK13_AUTH_PASSWORD" >"$login_body"
chmod 0600 "$login_body"
task13_run_logged "create authenticated smoke administration session" curl \
--connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" --max-time "$TASK13_CURL_MAX_TIME" \
--fail --silent --show-error --cookie-jar "$TASK13_ADMIN_COOKIE" \
-H "Origin: http://$frontend" -H 'content-type: application/json' --data-binary "@$login_body" \
"http://$frontend/api/auth/local/login"
me="$(curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" --max-time "$TASK13_CURL_MAX_TIME" \
--fail --silent --show-error --cookie "$TASK13_ADMIN_COOKIE" "http://$frontend/api/me")"
TASK13_ADMIN_CSRF="$(node -e 'const value=JSON.parse(process.argv[1]); if(typeof value.csrfToken!=="string") process.exit(1); process.stdout.write(value.csrfToken)' "$me")" \
|| task13_fail "authenticated smoke administration session lacks CSRF state"
}
task13_authenticated_get() {
local path="$1" frontend
frontend="$(task13_frontend_address)"
curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" --max-time "$TASK13_CURL_MAX_TIME" \
--fail --silent --show-error --cookie "$TASK13_ADMIN_COOKIE" "http://$frontend/api/$path"
}
task13_authenticated_post() {
local path="$1" frontend
frontend="$(task13_frontend_address)"
curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" --max-time 15 \
--fail --silent --show-error --cookie "$TASK13_ADMIN_COOKIE" \
-H "Origin: http://$frontend" -H "x-thothii-csrf: $TASK13_ADMIN_CSRF" \
-X POST "http://$frontend/api/$path"
}
task13_assert_local_restore_reauthentication() {
local frontend archive login_body cookie_before cookie_after me status_before status_after
frontend="$(task13_frontend_address)"
archive="$TASK13_TMP/local-restore-source.zip"
login_body="$TASK13_TMP/local-restore-login.json"
cookie_before="$TASK13_TMP/local-restore-before.cookies"
cookie_after="$TASK13_TMP/local-restore-after.cookies"
printf '{"username":"%s","password":"%s","remember":true}' \
"$TASK13_AUTH_ADMIN" "$TASK13_AUTH_PASSWORD" >"$login_body"
chmod 0600 "$login_body"
task13_run_logged "create pre-backup browser session" curl \
--connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" --max-time "$TASK13_CURL_MAX_TIME" \
--fail --silent --show-error --cookie-jar "$cookie_before" \
-H "Origin: http://$frontend" -H 'content-type: application/json' --data-binary "@$login_body" \
"http://$frontend/api/auth/local/login"
task13_compose_logged "stop local stack for backup" stop
task13_run_logged "create real default-custody backup" "$TASK13_THT" \
--installation "$TASK13_INSTALLATION" backup --output "$archive"
python3 - "$archive" <<'PY'
import json
import sys
import zipfile
with zipfile.ZipFile(sys.argv[1]) as archive:
manifest = json.loads(archive.read("manifest.json"))
volumes = [item["logical_name"] for item in manifest["volumes"]]
if volumes != ["embedding-models", "pi-state", "qdrant-data", "sessions", "settings", "workspace-registry", "workspace-secrets"]:
raise SystemExit(f"unexpected backup volume custody: {volumes}")
entries = manifest["entries"]
if any(item.get("logical_name") == "auth-state" or "/data/auth" in item.get("source_path", "") for item in entries):
raise SystemExit("default backup contains authentication runtime state")
auth = [item for item in entries if item["path"].startswith("authentication-secrets/")]
if len(auth) != 1 or not auth[0]["path"].endswith("-auth.yaml") or auth[0]["archived"]:
raise SystemExit("default backup auth custody is not an auth.yaml reference only")
if any(item["path"].endswith("users.yaml") for item in entries):
raise SystemExit("default backup contains users.yaml")
PY
task13_compose_start_logged "restart local stack before restore" up --detach --wait --wait-timeout 120
status_before="$(curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" --max-time "$TASK13_CURL_MAX_TIME" \
--silent --output /dev/null --write-out '%{http_code}' --cookie "$cookie_before" "http://$frontend/api/me")"
[[ "$status_before" == 200 ]] || task13_fail "pre-backup browser session did not survive an ordinary stop/start"
task13_run_logged "create post-backup browser session" curl \
--connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" --max-time "$TASK13_CURL_MAX_TIME" \
--fail --silent --show-error --cookie-jar "$cookie_after" \
-H "Origin: http://$frontend" -H 'content-type: application/json' --data-binary "@$login_body" \
"http://$frontend/api/auth/local/login"
me="$(curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" --max-time "$TASK13_CURL_MAX_TIME" \
--fail --silent --show-error --cookie "$cookie_after" "http://$frontend/api/me")"
node -e 'const value=JSON.parse(process.argv[1]); if(value.issuer!=="local"||value.session?.remembered!==true) process.exit(1)' "$me" \
|| task13_fail "post-backup browser session was not authenticated"
task13_compose_logged "seed valid pending OIDC state excluded from restore" exec -T core node --input-type=module -e '
const { loadConfig } = await import("/app/backend/dist/config.js");
const { createFileAuthSessionStore } = await import("/app/backend/dist/auth/session-store.js");
const config = loadConfig(process.env);
const revision = config.authentication.current().revision;
const store = createFileAuthSessionStore(config.authStateRoot);
await store.createOidcState({
nonce: "n".repeat(43), codeVerifier: "v".repeat(43), returnTo: "/",
authConfigRevision: revision, issuer: "https://pending.task13.invalid",
browserTransactionDigest: "d".repeat(64), browserTransactionTransport: "loopback_http",
});
'
task13_compose_logged "verify pre-restore authentication runtime population" exec -T core sh -ceu \
'test "$(find /data/auth/sessions -type f | wc -l | tr -d " ")" -ge 2 && test -n "$(find /data/auth/oidc -type f -name "*.json" -print -quit)"'
task13_run_logged "perform real running local production restore" "$TASK13_THT" \
--installation "$TASK13_INSTALLATION" restore "$archive" --yes
status_before="$(curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" --max-time "$TASK13_CURL_MAX_TIME" \
--silent --output /dev/null --write-out '%{http_code}' --cookie "$cookie_before" "http://$frontend/api/me")"
status_after="$(curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" --max-time "$TASK13_CURL_MAX_TIME" \
--silent --output /dev/null --write-out '%{http_code}' --cookie "$cookie_after" "http://$frontend/api/me")"
[[ "$status_before" == 401 && "$status_after" == 401 ]] \
|| task13_fail "restore did not force every independent browser session to reauthenticate"
task13_compose_logged "verify private empty restored auth state" exec -T core sh -ceu '
test "$(stat -c %a /data/auth)" = 700
test "$(stat -c %a /data/auth/sessions)" = 700
test "$(stat -c %a /data/auth/oidc)" = 700
test "$(stat -c %u /data/auth)" = "$(id -u)"
test -z "$(find /data/auth/sessions /data/auth/oidc -mindepth 1 -print -quit)"
'
task13_create_admin_session
}
task13_assert_server_oidc_restore_verification() {
local archive frontend status diagnostics restore_output restore_rc restore_cause
local rollback_output rollback_rc rollback_sentinel provider_label checkpoint_leftover
local registry_before registry_after integrity_output
archive="$TASK13_TMP/server-oidc-restore-source.zip"
frontend="$(task13_frontend_address)"
task13_compose_logged "seed valid server authentication runtime excluded from restore" exec -T core node --input-type=module -e '
const { loadConfig } = await import("/app/backend/dist/config.js");
const { createFileAuthSessionStore } = await import("/app/backend/dist/auth/session-store.js");
const config = loadConfig(process.env);
const revision = config.authentication.current().revision;
const store = createFileAuthSessionStore(config.authStateRoot);
await store.create({
principal: { issuer: "https://task13-fake-oidc:9443/application/o/task13/", subject: "restore-browser", roles: ["user"], permissions: ["session.use"], isAdmin: false },
method: "oidc", remembered: false, authConfigRevision: revision,
idleTtlMs: 60_000, absoluteTtlMs: 120_000,
});
await store.createOidcState({
nonce: "n".repeat(43), codeVerifier: "v".repeat(43), returnTo: "/",
authConfigRevision: revision, issuer: "https://task13-fake-oidc:9443/application/o/task13/",
browserTransactionDigest: "d".repeat(64), browserTransactionTransport: "https",
});
'
task13_compose_logged "stop server stack for OIDC restore" stop
rollback_sentinel="$TASK13_SERVER_DATA/task13-restore-rollback"
printf 'backup-state\n' >"$rollback_sentinel"
task13_run_logged "create real server default-custody backup" task13_server_tht \
backup --output "$archive"
printf 'current-state\n' >"$rollback_sentinel"
provider_label="$(docker container inspect --format '{{ index .Config.Labels "io.thothii.task13.run" }}' "$TASK13_OIDC_CONTAINER")"
[[ "$provider_label" == "$TASK13_RUN_ID" ]] || task13_fail "fake OIDC provider ownership changed before rollback injection"
task13_run_logged "inject post-mutation OIDC verification failure" docker stop "$TASK13_OIDC_CONTAINER"
rollback_output="$TASK13_TMP/server-oidc-rollback.out"
set +e
task13_server_tht restore "$archive" --yes \
>"$rollback_output" 2>&1
rollback_rc=$?
set -e
[[ "$rollback_rc" -ne 0 ]] || task13_fail "server restore unexpectedly passed with its OIDC provider unavailable"
grep -Eq 'restore verification doctor:|authentication diagnostics did not pass after restore' "$rollback_output" \
|| task13_fail "server restore rollback injection did not reach post-mutation authentication verification"
if grep -Fq "$TASK13_OIDC_CLIENT_SECRET" "$rollback_output" \
|| grep -Fq "$TASK13_AUTHENTIK_API_TOKEN" "$rollback_output"; then
task13_fail "failed server restore exposed fake-provider custody values"
fi
[[ "$(sudo -n -- cat -- "$rollback_sentinel")" == current-state ]] \
|| task13_fail "failed server restore did not roll back the server data bind"
checkpoint_leftover="$(task13_server_checkpoint_leftover)"
[[ -z "$checkpoint_leftover" ]] || task13_fail "failed server restore retained its private recovery checkpoint"
task13_compose_logged "verify failed restore cleared authentication runtime" \
run --rm --no-deps --no-TTY core sh -ceu '
test "$(stat -c %a /data/auth)" = 700
test "$(stat -c %a /data/auth/sessions)" = 700
test "$(stat -c %a /data/auth/oidc)" = 700
test "$(stat -c %u /data/auth)" = "$(id -u)"
test -z "$(find /data/auth/sessions /data/auth/oidc -mindepth 1 -print -quit)"
'
task13_run_logged "restore scoped fake OIDC provider after failure injection" docker start "$TASK13_OIDC_CONTAINER"
for _attempt in $(seq 1 30); do
if docker exec "$TASK13_OIDC_CONTAINER" node -e \
"fetch('https://task13-fake-oidc:9443/health',{signal:AbortSignal.timeout(3000)}).then(r=>{if(!r.ok)process.exit(1)}).catch(()=>process.exit(1))" \
>>"$TASK13_LOG" 2>&1; then
break
fi
sleep 1
done
docker exec "$TASK13_OIDC_CONTAINER" node -e \
"fetch('https://task13-fake-oidc:9443/health',{signal:AbortSignal.timeout(3000)}).then(r=>{if(!r.ok)process.exit(1)}).catch(()=>process.exit(1))" \
>>"$TASK13_LOG" 2>&1 || task13_log_failure "restored scoped fake OIDC provider readiness"
printf 'Task 13 server rollback injection passed: exit=%s; recovery checkpoint removed.\n' "$rollback_rc"
registry_before="$(task13_registry_filesystem_fingerprint "$TASK13_SERVER_REGISTRY")"
integrity_output="$TASK13_TMP/server-workspace-integrity.json"
if ! task13_compose run --rm --no-deps --no-TTY core \
node /app/backend/dist/operator-command.js workspace-integrity \
>"$integrity_output" 2>>"$TASK13_LOG"; then
task13_log_failure "stopped read-only workspace registry verification"
fi
node -e '
const value=JSON.parse(require("fs").readFileSync(process.argv[1], "utf8"));
const keys=Object.keys(value).sort().join(",");
if(keys!=="fingerprint,ready,state,workspaces"||value.ready!==true||value.state!=="uninitialized"||value.workspaces!==0||!/^sha256:[0-9a-f]{64}$/.test(value.fingerprint)) process.exit(1);
' "$integrity_output" || task13_fail "stopped registry inspector returned malformed or additional output"
registry_after="$(task13_registry_filesystem_fingerprint "$TASK13_SERVER_REGISTRY")"
[[ "$registry_after" == "$registry_before" ]] \
|| task13_fail "stopped restore verification mutated the workspace registry filesystem"
restore_output="$TASK13_TMP/server-oidc-restore.out"
set +e
task13_server_tht restore "$archive" --yes \
>"$restore_output" 2>&1
restore_rc=$?
set -e
if [[ "$restore_rc" -ne 0 ]]; then
restore_cause=restore-failed
if grep -Fq 'restore verification workspace: validate restored workspace registry' "$restore_output"; then
restore_cause=workspace-validator-rejected
fi
printf 'Task 13 stopped restore diagnostic: exit=%s cause=%s\n' \
"$restore_rc" "$restore_cause" >&2
task13_sanitize <"$restore_output" | tail -n 8 >&2
return "$restore_rc"
fi
checkpoint_leftover="$(task13_server_checkpoint_leftover)"
[[ -z "$checkpoint_leftover" ]] || task13_fail "successful server restore retained its private recovery checkpoint"
task13_compose_start_logged "start restored server stack" up --detach --wait --wait-timeout 120 core frontend
frontend="$(task13_frontend_address)"
status=""
for _attempt in $(seq 1 30); do
status="$(curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" --max-time "$TASK13_CURL_MAX_TIME" \
--silent --output /dev/null --write-out '%{http_code}' "http://$frontend/api/me" || true)"
[[ "$status" == 401 ]] && break
sleep 1
done
[[ "$status" == 401 ]] \
|| task13_fail "OIDC restore did not require browser reauthentication (HTTP ${status:-unavailable})"
task13_compose_logged "verify private empty server authentication state" exec -T core sh -ceu '
test "$(stat -c %a /data/auth)" = 700
test "$(stat -c %a /data/auth/sessions)" = 700
test "$(stat -c %a /data/auth/oidc)" = 700
test "$(stat -c %u /data/auth)" = "$(id -u)"
test -z "$(find /data/auth/sessions /data/auth/oidc -mindepth 1 -print -quit)"
'
diagnostics="$TASK13_TMP/server-auth-diagnostics-after-restore.json"
task13_server_tht auth check --json >"$diagnostics"
node -e 'const value=JSON.parse(require("fs").readFileSync(process.argv[1], "utf8")); if(value.mode!=="oidc"||value.ready!==true||value.checks.length!==1||value.checks[0].code!=="auth_ready") process.exit(1)' "$diagnostics" \
|| task13_fail "restored server did not retain strict fake-provider OIDC diagnostics"
}
task13_assert_runtime() {
local frontend expected_pi actual_pi core_id
frontend="$(task13_frontend_address)"
expected_pi="$(sed -n 's/^ARG PI_VERSION=//p' "$TASK13_ROOT/docker/core.Dockerfile" | head -n 1)"
actual_pi="$(task13_compose exec -T core pi --version | tr -d '\r\n')"
[[ -n "$expected_pi" && "$actual_pi" == "$expected_pi" ]] || task13_fail "embedded Pi version mismatch"
task13_run_logged "frontend health" curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" \
--max-time "$TASK13_CURL_MAX_TIME" --fail --silent --show-error "http://$frontend/"
task13_run_logged "same-origin core health" curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" \
--max-time "$TASK13_CURL_MAX_TIME" --fail --silent --show-error "http://$frontend/api/health"
task13_compose_logged "core non-root identity" exec -T core sh -ceu \
'test "$(id -u)" = 10001'
task13_compose_logged "embedded Pi executable" exec -T core sh -ceu \
'command -v pi >/dev/null'
task13_compose_logged "core Docker socket isolation" exec -T core sh -ceu \
'test ! -e /var/run/docker.sock'
task13_assert_local_auth_lifecycle
task13_create_admin_session
task13_authenticated_get workspace-registry/status >/dev/null \
|| task13_fail "authenticated workspace registry bootstrap failed"
task13_compose_logged "active workspace registry state" exec -T core sh -ceu \
'test -f /data/workspace-registry/state/active.json'
task13_compose_logged "mounted Pi auth readability" exec -T core sh -ceu \
'test -r /home/thoth/.pi/agent/auth.json'
task13_compose_logged "mounted application secret readability" exec -T core sh -ceu \
'test -r /run/secrets/thothii.secrets'
core_id="$(task13_core_id)"
[[ "$(docker inspect --format '{{ index .Config.Labels "io.thothii.task13.run" }}' "$core_id")" == "$TASK13_RUN_ID" ]] \
|| task13_fail "core lacks the explicit Task 13 resource label"
task13_assert_maintenance_auth_isolation
task13_run_logged "tht Pi doctor" "$TASK13_THT" --installation "$TASK13_INSTALLATION" pi doctor
}
task13_report_server_workspace_failure() {
local status="$1" response="$2"
printf 'authenticated server /api/workspaces returned HTTP %s\n' "$status" >&2
printf '%s\n' '--- sanitized server workspace response ---' >&2
if [[ -s "$response" ]]; then
tail -c 16384 "$response" | task13_sanitize >&2
else
printf '%s\n' '(empty response)' >&2
fi
printf '%s\n' '--- sanitized registry integrity probe ---' >&2
{
task13_compose exec -T core node --input-type=module -e '
const { loadConfig } = await import("/app/backend/dist/config.js");
const { WorkspaceRegistry } = await import("/app/backend/dist/workspaces/registry.js");
const registry = new WorkspaceRegistry(loadConfig(process.env).workspaceRegistry);
try {
const revisions = await registry.list();
for (const revision of revisions) await registry.read(revision.id);
console.log(JSON.stringify({ ok: true, revisions: revisions.length }));
} catch (error) {
console.log(JSON.stringify({
ok: false,
name: error instanceof Error ? error.name : "UnknownError",
code: error && typeof error === "object" && "code" in error ? error.code : "unknown",
message: error instanceof Error ? error.message : "Unknown registry failure",
}));
process.exitCode = 1;
}
' 2>&1 || printf '%s\n' '(registry integrity probe unavailable)'
} | tail -n 20 | task13_sanitize >&2
printf '%s\n' '--- sanitized core logs (last 100 lines) ---' >&2
{
task13_compose logs --no-color --tail 100 core 2>&1 \
|| printf '%s\n' '(core logs unavailable)'
} | tail -n 100 | task13_sanitize >&2
}
task13_assert_server_runtime() {
local frontend unauthenticated trusted_header_status session_status diagnostics status provider_requests core_id frontend_id
local expected_core_image expected_frontend_image
frontend="$(task13_frontend_address)"
task13_run_logged "server frontend health" curl \
--connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" --max-time "$TASK13_CURL_MAX_TIME" \
--fail --silent --show-error "http://$frontend/"
task13_run_logged "server same-origin core health" curl \
--connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" --max-time "$TASK13_CURL_MAX_TIME" \
--fail --silent --show-error "http://$frontend/api/health"
core_id="$(task13_core_id)"
frontend_id="$(task13_compose ps -q frontend)"
expected_core_image="$(docker image inspect --format '{{.Id}}' "$TASK13_CORE_IMAGE")"
expected_frontend_image="$(docker image inspect --format '{{.Id}}' "$TASK13_FRONTEND_IMAGE")"
[[ "$(docker inspect --format '{{.Image}}' "$core_id")" == "$expected_core_image" ]] \
|| task13_fail "server core did not use the smoke-built core image"
[[ "$(docker inspect --format '{{.Image}}' "$frontend_id")" == "$expected_frontend_image" ]] \
|| task13_fail "server frontend did not use the smoke-built frontend image"
task13_compose_logged "verify server runtime configuration and secret readability" exec -T core sh -ceu '
check_readable() { test -r "$1" || { printf "server precondition failed: unreadable %s\n" "$1" >&2; exit 1; }; }
check_readable /run/thothii-auth/CURRENT
projection_generation="$(node -e '\''
const fs = require("node:fs");
const selector = JSON.parse(fs.readFileSync("/run/thothii-auth/CURRENT", "utf8"));
if (selector.version !== 1 || selector.state !== "ready" ||
typeof selector.generation !== "string" || !/^[0-9a-f]{64}$/.test(selector.generation)) {
process.exit(1);
}
process.stdout.write(selector.generation);
'\'')" || { printf "server precondition failed: invalid authentication projection selector\n" >&2; exit 1; }
check_readable "/run/thothii-auth/generations/$projection_generation/auth.yaml"
check_readable "/run/thothii-auth/generations/$projection_generation/manifest.json"
test -d /data/auth || { printf "server precondition failed: missing /data/auth\n" >&2; exit 1; }
test -z "${AUTH_MODE+x}" || { printf "server precondition failed: AUTH_MODE must be unset\n" >&2; exit 1; }
test "$THT_SESSION_STORAGE" = postgres || { printf "server precondition failed: session storage\n" >&2; exit 1; }
check_readable /run/secrets/thothii.secrets
check_readable /run/secrets/session_runtime_password
check_readable /run/secrets/session_ca.pem
check_readable /app/harness/workspaces/server-sessions.yaml
'
task13_mount_fingerprint | grep -Fq '/data = bind :' \
|| task13_fail "server profile did not bind the disposable data root"
task13_mount_fingerprint | grep -Fq '/home/thoth/.pi = bind :' \
|| task13_fail "server profile did not bind the disposable Pi state root"
task13_mount_fingerprint | grep -Fq '/data/workspace-registry = bind :' \
|| task13_fail "server profile did not bind the disposable registry root"
task13_assert_maintenance_auth_isolation
unauthenticated="$(curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" \
--max-time "$TASK13_CURL_MAX_TIME" --silent --output /dev/null --write-out '%{http_code}' \
"http://$frontend/api/workspaces")"
[[ "$unauthenticated" == 401 ]] || task13_fail "server profile did not enforce OIDC authentication"
trusted_header_status="$(curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" \
--max-time "$TASK13_CURL_MAX_TIME" --silent --output /dev/null --write-out '%{http_code}' \
-H 'x-thoth-trusted-principal-issuer: task13-proxy' \
-H 'x-thoth-trusted-principal-subject: task13-user' \
-H 'x-thoth-trusted-principal-display-name: Task 13 User' \
-H 'x-thoth-trusted-is-admin: 0' \
"http://$frontend/api/workspaces")"
[[ "$trusted_header_status" == 401 ]] || task13_fail "server accepted retired trusted identity headers"
session_status="$(curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" \
--max-time "$TASK13_CURL_MAX_TIME" --silent --output "$TASK13_TMP/server-sessions.out" \
--write-out '%{http_code}' \
"http://$frontend/api/sessions")"
[[ "$session_status" == 401 ]] \
|| task13_fail "server session route did not fail closed before OIDC authentication"
if grep -Fq "$TASK13_SECRET_VALUE" "$TASK13_TMP/server-sessions.out"; then
task13_fail "server session failure exposed the fixture secret"
fi
status="$TASK13_TMP/server-auth-status.json"
task13_run_logged "server static OIDC status" task13_server_tht auth status --json
task13_server_tht auth status --json >"$status"
node -e 'const value=JSON.parse(require("fs").readFileSync(process.argv[1], "utf8")); if(value.state!=="ready"||value.equal!==true||!/^[0-9a-f]{64}$/.test(value.generation)||value.canonicalRevision!==`sha256:${value.generation}`) process.exit(1)' "$status" \
|| task13_fail "server authentication projection status was not valid"
diagnostics="$TASK13_TMP/server-auth-diagnostics.json"
task13_run_logged "server live OIDC diagnostics" task13_server_tht auth check --json
task13_server_tht auth check --json >"$diagnostics"
node -e 'const value=JSON.parse(require("fs").readFileSync(process.argv[1], "utf8")); if(value.mode!=="oidc"||value.ready!==true||value.checks.length!==1||value.checks[0].code!=="auth_ready") process.exit(1)' "$diagnostics" \
|| task13_fail "scoped fake OIDC provider did not pass live production diagnostics"
provider_requests="$(docker logs "$TASK13_OIDC_CONTAINER" 2>>"$TASK13_LOG")"
[[ "$(grep -Fc 'group:task13-users' <<<"$provider_requests")" -ge 1 ]] \
|| task13_fail "live OIDC diagnostics did not verify the mandatory user group"
[[ "$(grep -Fc 'group:task13-admins' <<<"$provider_requests")" -ge 1 ]] \
|| task13_fail "live OIDC diagnostics did not verify the mandatory administrator group"
if grep -Fq 'group:task13-unrelated' <<<"$provider_requests" \
|| grep -Fq 'task13-unrelated' "$diagnostics"; then
task13_fail "live OIDC diagnostics queried or warned about an unrelated group"
fi
if grep -Fq "$TASK13_OIDC_CLIENT_SECRET" "$diagnostics" || grep -Fq "$TASK13_AUTHENTIK_API_TOKEN" "$diagnostics"; then
task13_fail "OIDC diagnostics exposed a fixture secret"
fi
}
task13_registry_status() {
task13_authenticated_get workspace-registry/status
}
task13_registry_head() {
sed -n 's/.*"head":"\([0-9a-f][0-9a-f]*\)".*/\1/p'
}
task13_active_registry_head() {
task13_compose exec -T core sed -n \
's/.*"head":"\([0-9a-f][0-9a-f]*\)".*/\1/p' \
/data/workspace-registry/state/active.json
}
task13_assert_sentinels() {
task13_compose exec -T core sh -ceu '
test "$(cat /data/settings/task13-settings)" = settings-preserved
test "$(cat /data/sessions/task13-session)" = sessions-preserved
test "$(cat /home/thoth/.pi/task13-pi-state)" = pi-state-preserved
test -f /data/workspace-registry/state/active.json
'
}
task13_mount_fingerprint() {
docker inspect --format '{{range .Mounts}}{{println .Destination "=" .Type ":" .Name}}{{end}}' "$(task13_core_id)" \
| LC_ALL=C sort
}
task13_service_mount_fingerprint() {
local service="$1"
docker inspect --format '{{range .Mounts}}{{println .Destination "=" .Type ":" .Name}}{{end}}' \
"$(task13_compose ps -q "$service")" | LC_ALL=C sort
}
task13_prepare_persistence() {
task13_compose exec -T core sh -ceu '
printf %s settings-preserved > /data/settings/task13-settings
printf %s sessions-preserved > /data/sessions/task13-session
printf %s pi-state-preserved > /home/thoth/.pi/task13-pi-state
'
TASK13_INITIAL_MOUNTS="$(task13_mount_fingerprint)"
TASK13_INITIAL_HEAD="$(task13_active_registry_head)"
[[ "$TASK13_INITIAL_HEAD" =~ ^[0-9a-f]{40}$ ]] || task13_fail "initial registry head is invalid"
task13_authenticated_get workspaces \
| grep -Fq 'Task 13 Smoke' || task13_fail "initial workspace is unavailable"
}
task13_registry_lifecycle() {
local offline_status offline_head valid_head evidence_head repaired_head
printf '== Recreate offline and retain the validated registry snapshot ==
'
task13_write_environment /fixtures/offline.git
task13_compose_logged "offline Compose recreation" up --detach --force-recreate --wait --wait-timeout 120
offline_status="$(task13_registry_status)"
offline_head="$(printf '%s' "$offline_status" | task13_registry_head)"
[[ "$offline_head" == "$TASK13_INITIAL_HEAD" ]] || task13_fail "offline recreation changed registry head"
grep -Fq '"degraded":true' <<<"$offline_status" || task13_fail "offline recreation did not report degraded mode"
task13_assert_sentinels
[[ "$(task13_mount_fingerprint)" == "$TASK13_INITIAL_MOUNTS" ]] || task13_fail "offline recreation changed volume identity"
printf '== Pull a valid catalog+descriptor metadata update ==
'
task13_replace_once "$TASK13_SEED/$TASK13_WORKSPACE_ID/workspace.yaml" 'name: Task 13 Smoke' 'name: Task 13 Smoke Updated'
task13_replace_once "$TASK13_SEED/thoth-workspaces.yaml" 'name: Task 13 Smoke' 'name: Task 13 Smoke Updated'
task13_commit_registry_change 'Update Task 13 workspace metadata'
task13_write_environment /fixtures/remote.git
task13_compose_logged "online Compose recreation" up --detach --force-recreate --wait --wait-timeout 120
task13_authenticated_post workspace-registry/pull >/dev/null
task13_authenticated_get workspaces | grep -Fq 'Task 13 Smoke Updated' || task13_fail "valid Git update was not activated"
valid_head="$(task13_active_registry_head)"
[[ "$valid_head" =~ ^[0-9a-f]{40}$ && "$valid_head" != "$TASK13_INITIAL_HEAD" ]] || task13_fail "valid Git update did not advance the registry head"
TASK13_INITIAL_HEAD="$valid_head"
task13_assert_sentinels
printf '== Pull a content-only Git Evidence update ==
'
printf 'guide v2\n' >"$TASK13_SEED/$TASK13_WORKSPACE_ID/evidence/guide.md"
task13_commit_registry_change 'Update Task 13 workspace evidence only'
task13_authenticated_post workspace-registry/pull >/dev/null
evidence_head="$(task13_active_registry_head)"
[[ "$evidence_head" =~ ^[0-9a-f]{40}$ && "$evidence_head" != "$valid_head" ]] || task13_fail "content-only Git Evidence update did not advance the registry head"
TASK13_INITIAL_HEAD="$evidence_head"
task13_assert_sentinels
printf '== Reject catalog/descriptor metadata mismatch and retain the valid snapshot ==
'
task13_replace_once "$TASK13_SEED/thoth-workspaces.yaml" 'name: Task 13 Smoke Updated' 'name: Task 13 Smoke Drift'
task13_commit_registry_change 'Break Task 13 workspace metadata parity'
if task13_authenticated_post workspace-registry/pull >>"$TASK13_LOG" 2>&1; then
task13_fail "registry accepted catalog/descriptor metadata mismatch"
fi
[[ "$(task13_active_registry_head)" == "$TASK13_INITIAL_HEAD" ]] || task13_fail "metadata mismatch replaced the valid registry head"
task13_authenticated_get workspaces | grep -Fq 'Task 13 Smoke Updated' || task13_fail "metadata mismatch displaced the valid workspace"
task13_replace_once "$TASK13_SEED/thoth-workspaces.yaml" 'name: Task 13 Smoke Drift' 'name: Task 13 Smoke Updated'
task13_commit_registry_change 'Restore Task 13 workspace metadata parity'
task13_authenticated_post workspace-registry/pull >/dev/null
repaired_head="$(task13_active_registry_head)"
[[ "$repaired_head" =~ ^[0-9a-f]{40}$ && "$repaired_head" != "$TASK13_INITIAL_HEAD" ]] || task13_fail "metadata repair did not restore a fresh valid registry head"
TASK13_INITIAL_HEAD="$repaired_head"
printf '== Reject orphan descriptor directories not listed in the catalog ==
'
mkdir -p "$TASK13_SEED/orphan"
cp "$TASK13_SEED/$TASK13_WORKSPACE_ID/workspace.yaml" "$TASK13_SEED/orphan/workspace.yaml"
task13_replace_once "$TASK13_SEED/orphan/workspace.yaml" 'id: task13-smoke' 'id: orphan'
task13_replace_once "$TASK13_SEED/orphan/workspace.yaml" 'name: Task 13 Smoke Updated' 'name: Orphan Workspace'
task13_replace_once "$TASK13_SEED/orphan/workspace.yaml" 'collection: task13-smoke' 'collection: orphan'
task13_replace_once "$TASK13_SEED/orphan/workspace.yaml" 'uri: task13-smoke/evidence' 'uri: orphan/evidence'
mkdir -p "$TASK13_SEED/orphan/evidence"
printf 'orphan guide\n' >"$TASK13_SEED/orphan/evidence/guide.md"
task13_commit_registry_change 'Add orphan Task 13 workspace directory'
if task13_authenticated_post workspace-registry/pull >>"$TASK13_LOG" 2>&1; then
task13_fail "registry accepted orphan Task 13 descriptor directory"
fi
[[ "$(task13_active_registry_head)" == "$TASK13_INITIAL_HEAD" ]] || task13_fail "orphan descriptor directory replaced the valid registry head"
task13_authenticated_get workspaces | grep -Fq 'Task 13 Smoke Updated' || task13_fail "orphan descriptor displaced the valid workspace"
rm -rf "$TASK13_SEED/orphan"
task13_commit_registry_change 'Remove orphan Task 13 workspace directory'
task13_authenticated_post workspace-registry/pull >/dev/null
TASK13_INITIAL_HEAD="$(task13_active_registry_head)"
printf '== Reject the retired flat workspace layout and retain the valid snapshot ==
'
mkdir -p "$TASK13_SEED/workspaces" "$TASK13_SEED/workspace-content/$TASK13_WORKSPACE_ID/evidence"
cp "$TASK13_ROOT/scripts/fixtures/workspace-registry-task13.yaml" "$TASK13_SEED/workspaces/$TASK13_WORKSPACE_ID.yaml"
printf 'legacy guide\n' >"$TASK13_SEED/workspace-content/$TASK13_WORKSPACE_ID/evidence/guide.md"
task13_commit_registry_change 'Reintroduce retired flat Task 13 workspace layout'
if task13_authenticated_post workspace-registry/pull >>"$TASK13_LOG" 2>&1; then
task13_fail "registry accepted the retired flat Task 13 workspace layout"
fi
[[ "$(task13_active_registry_head)" == "$TASK13_INITIAL_HEAD" ]] || task13_fail "retired flat workspace layout replaced the valid registry head"
task13_authenticated_get workspaces | grep -Fq 'Task 13 Smoke Updated' || task13_fail "retired flat workspace layout displaced the valid workspace"
task13_assert_sentinels
}
task13_prepare_bad_candidate() {
task13_run_logged "pull pinned stopped-core candidate" docker image pull "$TASK13_BAD_CANDIDATE_IMAGE"
TASK13_BAD_CANDIDATE_ID="$(docker image inspect --format '{{.Id}}' "$TASK13_BAD_CANDIDATE_IMAGE")"
[[ "$TASK13_BAD_CANDIDATE_ID" =~ ^sha256:[0-9a-f]{64}$ ]] \
|| task13_fail "bad candidate image identity was not resolved"
task13_record_image_evidence "$TASK13_BAD_CANDIDATE_IMAGE" rollback-candidate
task13_run_logged "prove bad candidate exits" docker run \
--name "$TASK13_BAD_CANDIDATE_CONTAINER" \
--label "io.thothii.task13.run=$TASK13_RUN_ID" \
"$TASK13_BAD_CANDIDATE_IMAGE"
[[ "$(docker container inspect --format '{{.State.Running}}:{{.State.ExitCode}}' \
"$TASK13_BAD_CANDIDATE_CONTAINER")" == false:0 ]] \
|| task13_fail "bad candidate did not reach the guaranteed stopped state"
task13_remove_labeled_container "$TASK13_BAD_CANDIDATE_CONTAINER"
}
task13_update_rollback() {
local before_image before_mounts before_head output rc phase after_image after_mounts after_head
printf '== Inject a bad pinned Pi candidate and prove automatic rollback ==\n'
task13_prepare_bad_candidate
before_image="$(docker inspect --format '{{.Image}}' "$(task13_core_id)")"
TASK13_PREVIOUS_IMAGE_ID="$before_image"
before_mounts="$(task13_mount_fingerprint)"
before_head="$(task13_active_registry_head)"
output="$TASK13_TMP/tht-update.out"
set +e
"$TASK13_THT" --installation "$TASK13_INSTALLATION" pi update \
--version "$TASK13_BAD_PI_VERSION" --source pull --image "$TASK13_BAD_CANDIDATE_IMAGE" --yes \
>"$output" 2>&1
rc=$?
set -e
[[ "$rc" -ne 0 ]] || task13_fail "bad Pi candidate unexpectedly passed update verification"
if grep -Fq "$TASK13_SECRET_VALUE" "$output"; then
task13_fail "tht update output exposed the fixture secret"
fi
grep -Fq 'previous core image was restored' "$output" \
|| { task13_sanitize <"$output" >&2; task13_fail "tht did not report automatic rollback"; }
[[ -f "$TASK13_UPDATE_STATE" ]] || task13_fail "tht update state was not persisted"
phase="$(sed -n 's/.*"phase": "\([^"]*\)".*/\1/p' "$TASK13_UPDATE_STATE" | head -n 1)"
[[ "$phase" == rolled_back ]] || task13_fail "update state phase is not rolled_back"
if grep -Fq "$TASK13_SECRET_VALUE" "$TASK13_UPDATE_STATE"; then
task13_fail "update state exposed the fixture secret"
fi
task13_compose_files
after_image="$(docker inspect --format '{{.Image}}' "$(task13_core_id)")"
after_mounts="$(task13_mount_fingerprint)"
after_head="$(task13_active_registry_head)"
[[ "$after_image" == "$before_image" ]] || task13_fail "rollback did not restore the previous core image"
[[ "$after_mounts" == "$before_mounts" ]] || task13_fail "rollback changed persistence volume identity"
[[ "$after_head" == "$before_head" ]] || task13_fail "rollback changed the active registry revision"
task13_assert_sentinels
task13_run_logged "post-rollback tht doctor" \
"$TASK13_THT" --installation "$TASK13_INSTALLATION" pi doctor
task13_authenticated_get workspaces \
| grep -Fq 'Task 13 Smoke' || task13_fail "rollback lost the active workspace"
}
task13_remove_labeled_container() {
local name="$1" label
[[ -n "$name" ]] || return 0
if ! docker container inspect "$name" >/dev/null 2>&1; then
return 0
fi
label="$(docker container inspect --format '{{ index .Config.Labels "io.thothii.task13.run" }}' "$name")"
[[ "$label" == "$TASK13_RUN_ID" ]] || {
printf 'refusing to remove foreign container %s\n' "$name" >&2
return 1
}
task13_bounded "$TASK13_CLEANUP_TIMEOUT" "remove owned container" \
docker container rm --force "$name" >/dev/null
}
task13_remove_labeled_image() {
local reference="$1" label
[[ -n "$reference" ]] || return 0
if ! docker image inspect "$reference" >/dev/null 2>&1; then
return 0
fi
label="$(docker image inspect --format '{{ index .Config.Labels "io.thothii.task13.run" }}' "$reference")"
[[ "$label" == "$TASK13_RUN_ID" ]] || {
printf 'refusing to remove foreign image %s\n' "$reference" >&2
return 1
}
task13_bounded "$TASK13_CLEANUP_TIMEOUT" "remove owned image" \
docker image rm "$reference" >/dev/null
}
task13_remove_transaction_image() {
local reference="$1" expected_id="$2" actual_id
[[ -n "$reference" ]] || return 0
if ! docker image inspect "$reference" >/dev/null 2>&1; then
return 0
fi
if [[ ! "$reference" =~ ^thothii-core:tht-[0-9a-f]{16}-(candidate|previous)$ \
|| ! "$expected_id" =~ ^sha256:([0-9a-f]{64}|owned)$ ]]; then
printf 'refusing to remove invalid transaction image reference %s\n' "$reference" >&2
return 1
fi
actual_id="$(docker image inspect --format '{{.Id}}' "$reference")"
[[ "$actual_id" == "$expected_id" ]] || {
printf 'refusing to remove foreign transaction image %s\n' "$reference" >&2
return 1
}
task13_bounded "$TASK13_CLEANUP_TIMEOUT" "remove owned transaction image" \
docker image rm "$reference" >/dev/null
}
task13_assert_project_ownership() {
local kind id ids label
for kind in container volume network; do
if [[ "$kind" == container ]]; then
if ! ids="$(docker container ls -aq --filter "label=com.docker.compose.project=$TASK13_PROJECT")"; then
task13_fail "could not enumerate Compose project container resources"
return 1
fi
elif ! ids="$(docker "$kind" ls -q --filter "label=com.docker.compose.project=$TASK13_PROJECT")"; then
task13_fail "could not enumerate Compose project $kind resources"
return 1
fi
while IFS= read -r id; do
[[ -n "$id" ]] || continue
case "$kind" in
container)
if ! label="$(docker container inspect --format '{{ index .Config.Labels "io.thothii.task13.run" }}' "$id")"; then
task13_fail "could not inspect Compose project container resource"
return 1
fi
;;
volume)
if ! label="$(docker volume inspect --format '{{ index .Labels "io.thothii.task13.run" }}' "$id")"; then
task13_fail "could not inspect Compose project volume resource"
return 1
fi
;;
network)
if ! label="$(docker network inspect --format '{{ index .Labels "io.thothii.task13.run" }}' "$id")"; then
task13_fail "could not inspect Compose project network resource"
return 1
fi
;;
esac
if [[ "$label" != "$TASK13_RUN_ID" ]]; then
task13_fail "Compose project contains a foreign $kind resource"
return 1
fi
done <<<"$ids"
done
}
task13_assert_built_image_ownership() {
local image label
for image in "$TASK13_CORE_IMAGE" "$TASK13_FRONTEND_IMAGE"; do
label="$(docker image inspect --format '{{ index .Config.Labels "io.thothii.task13.run" }}' "$image")"
[[ "$label" == "$TASK13_RUN_ID" ]] || task13_fail "built image lacks the Task 13 run label"
done
}
task13_reclaim_server_fixture_ownership() {
local host_uid host_gid
[[ "${TASK13_PROFILE:-local}" == server ]] || return 0
[[ -n "${TASK13_TMP:-}" && -d "$TASK13_TMP" ]] || return 0
[[ "${TASK13_TMP%/*}" == "${TASK13_TMP_PARENT:-}" \
&& "${TASK13_TMP##*/}" == thothii-task13.* ]] \
|| task13_fail "refusing to reclaim an unexpected server fixture path"
[[ -n "${TASK13_CONTROL_DIR:-}" \
&& "$TASK13_CONTROL_DIR" == "$TASK13_ROOT/.tht/$TASK13_PROJECT" \
&& "$TASK13_PROJECT" =~ ^thothii-[0-9a-f]{12}$ ]] \
|| task13_fail "refusing to reclaim an unexpected server control path"
host_uid="$(id -u)"
host_gid="$(id -g)"
task13_bounded "$TASK13_CLEANUP_TIMEOUT" "reclaim server fixture ownership" \
sudo -n -- chown -hR "$host_uid:$host_gid" "$TASK13_TMP" "$TASK13_CONTROL_DIR"
}
task13_cleanup() {
local original_rc="$1" cleanup_rc=0 transaction="" leftovers="" image_id=""
set +e
if [[ "$original_rc" -ne 0 && -n "${TASK13_PROJECT:-}" \
&& -s "${TASK13_IMAGE_EVIDENCE_RECORDS:-}" && -n "${TASK13_IMAGE_EVIDENCE_OUTPUT:-}" ]]; then
if ! task13_capture_failed_image_evidence >>"${TASK13_LOG:-/dev/null}" 2>&1; then
rm -f "$TASK13_IMAGE_EVIDENCE_OUTPUT"
fi
fi
if [[ "$original_rc" -ne 0 && "${TASK13_FAILURE_LOGGED:-0}" -eq 0 ]] \
&& [[ -n "${TASK13_LOG:-}" && -f "$TASK13_LOG" ]]; then
printf '%s\n' '--- sanitized Task 13 diagnostic log ---' >&2
tail -n 200 "$TASK13_LOG" | task13_sanitize >&2
fi
task13_remove_labeled_container "${TASK13_BAD_CANDIDATE_CONTAINER:-}" || cleanup_rc=1
task13_remove_labeled_container "${TASK13_AUTH_PROJECTION_CONTAINER:-}" || cleanup_rc=1
task13_remove_labeled_container "${TASK13_PI_PROJECTION_CONTAINER:-}" || cleanup_rc=1
task13_remove_labeled_container "${TASK13_REGISTRY_PROJECTION_CONTAINER:-}" || cleanup_rc=1
task13_remove_labeled_container "${TASK13_LLM_CONTAINER:-}" || cleanup_rc=1
task13_remove_labeled_container "${TASK13_OIDC_CONTAINER:-}" || cleanup_rc=1
if [[ -n "${TASK13_PROJECT:-}" && -n "${TASK13_ROOT:-}" && -f "${TASK13_OVERRIDE:-}" ]]; then
if task13_assert_project_ownership >>"${TASK13_LOG:-/dev/null}" 2>&1; then
task13_compose_files
if ! task13_bounded "$TASK13_CLEANUP_TIMEOUT" "stop owned Compose project" \
task13_compose_invoke "${TASK13_COMPOSE[@]}" down --volumes --remove-orphans --timeout 10 \
>>"${TASK13_LOG:-/dev/null}" 2>&1; then
cleanup_rc=1
fi
else
cleanup_rc=1
fi
fi
if ! {
task13_reclaim_server_fixture_ownership
} >>"${TASK13_LOG:-/dev/null}" 2>&1; then
cleanup_rc=1
fi
if [[ -f "${TASK13_UPDATE_STATE:-}" ]]; then
transaction="$(sed -n 's/.*"transaction": "\([^"]*\)".*/\1/p' "$TASK13_UPDATE_STATE" | head -n 1)"
fi
if [[ -n "$transaction" ]]; then
task13_remove_transaction_image "thothii-core:tht-$transaction-candidate" \
"${TASK13_BAD_CANDIDATE_ID:-}" >>"${TASK13_LOG:-/dev/null}" 2>&1 || cleanup_rc=1
task13_remove_transaction_image "thothii-core:tht-$transaction-previous" \
"${TASK13_PREVIOUS_IMAGE_ID:-}" >>"${TASK13_LOG:-/dev/null}" 2>&1 || cleanup_rc=1
fi
for image in \
"${TASK13_FRONTEND_IMAGE:-}" \
"${TASK13_CORE_IMAGE:-}"; do
[[ -n "$image" ]] || continue
task13_remove_labeled_image "$image" >>"${TASK13_LOG:-/dev/null}" 2>&1 || cleanup_rc=1
done
if [[ -n "${TASK13_RUN_ID:-}" ]]; then
while IFS= read -r image_id; do
[[ -n "$image_id" ]] || continue
task13_remove_labeled_image "$image_id" >>"${TASK13_LOG:-/dev/null}" 2>&1 || cleanup_rc=1
done < <(docker image ls -q --filter "label=io.thothii.task13.run=$TASK13_RUN_ID" | sort -u)
fi
if [[ -n "${TASK13_CONTROL_DIR:-}" ]]; then
if [[ "$TASK13_CONTROL_DIR" == "$TASK13_ROOT/.tht/$TASK13_PROJECT" \
&& "$TASK13_PROJECT" =~ ^thothii-[0-9a-f]{12}$ ]]; then
rm -rf "$TASK13_CONTROL_DIR"
else
cleanup_rc=1
fi
fi
if [[ -n "${TASK13_RUN_ID:-}" ]]; then
leftovers="$(docker container ls -aq --filter "label=io.thothii.task13.run=$TASK13_RUN_ID")"
leftovers+="$(docker volume ls -q --filter "label=io.thothii.task13.run=$TASK13_RUN_ID")"
leftovers+="$(docker network ls -q --filter "label=io.thothii.task13.run=$TASK13_RUN_ID")"
leftovers+="$(docker image ls -q --filter "label=io.thothii.task13.run=$TASK13_RUN_ID")"
[[ -z "$leftovers" ]] || cleanup_rc=1
fi
if [[ -n "${TASK13_TMP:-}" && -d "$TASK13_TMP" ]]; then
if [[ "${TASK13_TMP%/*}" == "${TASK13_TMP_PARENT:-}" \
&& "${TASK13_TMP##*/}" == thothii-task13.* ]]; then
rm -rf "$TASK13_TMP"
else
cleanup_rc=1
fi
fi
if [[ "$cleanup_rc" -eq 0 ]]; then
printf 'Task 13 cleanup proof: no labeled containers, volumes, networks, or images remain for %s.\n' \
"${TASK13_RUN_ID:-unknown}"
else
printf 'Task 13 cleanup proof failed for %s.\n' "${TASK13_RUN_ID:-unknown}" >&2
fi
trap - EXIT
if [[ "$original_rc" -ne 0 ]]; then
exit "$original_rc"
fi
exit "$cleanup_rc"
}
task13_self_test_sanitizer() {
local input output leaked
TASK13_SECRET_VALUE="fixture-known-secret"
input="$(printf '%s\n' \
'fixture-known-secret' \
'password=plain-secret' \
'{"api_key":"json-secret"}' \
'Authorization: Bearer bearer-secret' \
'https://alice:url-secret@example.invalid/repo.git')"
output="$(printf '%s\n' "$input" | task13_sanitize)"
for leaked in fixture-known-secret plain-secret json-secret bearer-secret url-secret; do
if grep -Fq "$leaked" <<<"$output"; then
task13_fail "sanitizer leaked $leaked"
fi
done
[[ "$(grep -Fc '[REDACTED]' <<<"$output")" -eq 5 ]] \
|| task13_fail "sanitizer did not redact every credential form"
}
task13_self_test_server_workspace_diagnostics() {
local response output count i=1
response="$(mktemp "${TMPDIR:-/tmp}/thothii-task13-server-response.XXXXXX")"
TASK13_SECRET_VALUE="fixture-known-secret"
printf '%s\n' \
'{"error":"workspace_invalid","detail":"password=fixture-known-secret"}' >"$response"
task13_compose() {
if [[ "$*" == "exec -T core node --input-type=module -e "* ]]; then
printf '%s\n' \
'{"name":"WorkspaceRegistryError","code":"workspace_invalid","message":"Workspace snapshot integrity check failed"}'
return 0
fi
[[ "$*" == "logs --no-color --tail 100 core" ]] \
|| task13_fail "server diagnostics requested an unexpected Compose command"
while [[ "$i" -le 150 ]]; do
printf 'core-log-%03d token=fixture-known-secret\n' "$i"
i=$((i + 1))
done
}
if ! output="$(task13_report_server_workspace_failure 400 "$response" 2>&1)"; then
unset -f task13_compose
rm -f "$response"
task13_fail "server workspace diagnostics could not be captured"
fi
unset -f task13_compose
rm -f "$response"
[[ "$output" == *'authenticated server /api/workspaces returned HTTP 400'* ]] \
|| task13_fail "server diagnostics omit the unexpected HTTP status"
[[ "$output" == *'workspace_invalid'* ]] \
|| task13_fail "server diagnostics omit the generic response"
[[ "$output" == *'Workspace snapshot integrity check failed'* ]] \
|| task13_fail "server diagnostics omit the bounded internal registry reason"
[[ "$output" == *'core-log-051'* && "$output" != *'core-log-050'* ]] \
|| task13_fail "server diagnostics do not bound core logs to the last 100 lines"
count="$(grep -Ec '^core-log-[0-9]{3}' <<<"$output")"
[[ "$count" -eq 100 ]] || task13_fail "server diagnostics emitted $count core log lines"
[[ "$output" != *'fixture-known-secret'* ]] \
|| task13_fail "server diagnostics leaked the fixture secret"
[[ "$(grep -Fc '[REDACTED]' <<<"$output")" -ge 101 ]] \
|| task13_fail "server diagnostics did not sanitize response and core logs"
}
task13_self_test_core_startup_diagnostics() {
local output
TASK13_SECRET_VALUE="fixture-known-secret"
task13_compose() {
case "$*" in
"ps --all -q core") printf '%s\n' 'task13-core-id' ;;
"logs --no-color --tail 100 core")
printf '%s\n' \
'Error: EACCES: permission denied, mkdir /data/auth/sessions' \
'password=plain-secret token=fixture-known-secret' \
'secret path: /run/secrets/private-token' \
' at createFileAuthSessionStore (/app/backend/dist/auth/session-store.js:101:9)'
;;
*) task13_fail "startup diagnostics requested an unexpected Compose command: $*" ;;
esac
}
docker() {
[[ "$*" == "inspect --format {{.State.Status}}:{{.State.ExitCode}} task13-core-id" ]] \
|| task13_fail "startup diagnostics requested an unexpected Docker command: $*"
printf '%s\n' 'exited:1'
}
output="$(task13_report_core_startup_failure 2>&1)"
unset -f task13_compose docker
[[ "$output" == 'Core startup cause: authentication state storage is unavailable (exit code 1).' ]] \
|| task13_fail "startup diagnostics emitted a non-allowlisted cause: $output"
for leaked in EACCES permission /data/auth /run/secrets plain-secret fixture-known-secret \
createFileAuthSessionStore session-store.js; do
[[ "$output" != *"$leaked"* ]] || task13_fail "startup diagnostics leaked $leaked"
done
}
task13_self_test_cleanup_ownership() {
local calls foreign_error owned_name foreign_name
calls="$(mktemp "${TMPDIR:-/tmp}/thothii-task13-cleanup-contract.XXXXXX")"
foreign_error="$calls.foreign-error"
owned_name="task13-owned-contract"
foreign_name="task13-foreign-contract"
TASK13_RUN_ID="task13-contract-run"
docker() {
printf '%s\n' "$*" >>"$calls"
if [[ "$1 $2" == "container inspect" ]]; then
if [[ "$3" == "--format" ]]; then
if [[ "${*: -1}" == "$owned_name" ]]; then
printf '%s\n' "$TASK13_RUN_ID"
else
printf '%s\n' 'some-other-run'
fi
fi
return 0
fi
[[ "$1 $2" == "container rm" ]]
}
if task13_remove_labeled_container "$foreign_name" 2>"$foreign_error"; then
unset -f docker
rm -f "$calls" "$foreign_error"
task13_fail "cleanup accepted a foreign-labeled container"
fi
if grep -Fq "container rm --force $foreign_name" "$calls"; then
unset -f docker
rm -f "$calls" "$foreign_error"
task13_fail "cleanup attempted to remove a foreign-labeled container"
fi
grep -Fq "refusing to remove foreign container $foreign_name" "$foreign_error" \
|| task13_fail "cleanup refusal was not explicit"
task13_remove_labeled_container "$owned_name"
[[ "$(grep -Fc "container rm --force $owned_name" "$calls")" -eq 1 ]] \
|| task13_fail "cleanup did not remove exactly the owned container"
unset -f docker
rm -f "$calls" "$foreign_error"
}
task13_self_test_image_cleanup_ownership() {
local calls foreign_error owned_ref foreign_ref
calls="$(mktemp "${TMPDIR:-/tmp}/thothii-task13-image-cleanup-contract.XXXXXX")"
foreign_error="$calls.foreign-error"
owned_ref="task13-owned-contract:local"
foreign_ref="task13-foreign-contract:local"
TASK13_RUN_ID="task13-contract-run"
if ! declare -F task13_remove_labeled_image >/dev/null; then
rm -f "$calls" "$foreign_error"
task13_fail "image cleanup ownership guard is missing"
fi
docker() {
printf '%s\n' "$*" >>"$calls"
if [[ "$1 $2" == "image inspect" ]]; then
if [[ "$3" == "--format" ]]; then
if [[ "${*: -1}" == "$owned_ref" ]]; then
printf '%s\n' "$TASK13_RUN_ID"
else
printf '%s\n' 'some-other-run'
fi
fi
return 0
fi
[[ "$1 $2" == "image rm" ]]
}
if task13_remove_labeled_image "$foreign_ref" 2>"$foreign_error"; then
unset -f docker
rm -f "$calls" "$foreign_error"
task13_fail "cleanup accepted a foreign-labeled image"
fi
if grep -Fq "image rm $foreign_ref" "$calls"; then
unset -f docker
rm -f "$calls" "$foreign_error"
task13_fail "cleanup attempted to remove a foreign-labeled image"
fi
grep -Fq "refusing to remove foreign image $foreign_ref" "$foreign_error" \
|| task13_fail "image cleanup refusal was not explicit"
task13_remove_labeled_image "$owned_ref"
[[ "$(grep -Fc "image rm $owned_ref" "$calls")" -eq 1 ]] \
|| task13_fail "cleanup did not remove exactly the owned image reference"
unset -f docker
rm -f "$calls" "$foreign_error"
}
task13_self_test_transaction_image_cleanup() {
local calls foreign_error owned_ref foreign_ref
calls="$(mktemp "${TMPDIR:-/tmp}/thothii-task13-transaction-cleanup-contract.XXXXXX")"
foreign_error="$calls.foreign-error"
owned_ref="thothii-core:tht-0123456789abcdef-candidate"
foreign_ref="thothii-core:tht-fedcba9876543210-candidate"
if ! declare -F task13_remove_transaction_image >/dev/null; then
rm -f "$calls" "$foreign_error"
task13_fail "transaction image cleanup guard is missing"
fi
docker() {
printf '%s\n' "$*" >>"$calls"
if [[ "$1 $2" == "image inspect" ]]; then
if [[ "$3" == "--format" ]]; then
if [[ "${*: -1}" == "$owned_ref" ]]; then
printf '%s\n' 'sha256:owned'
else
printf '%s\n' 'sha256:foreign'
fi
fi
return 0
fi
[[ "$1 $2" == "image rm" ]]
}
if task13_remove_transaction_image "$foreign_ref" 'sha256:owned' 2>"$foreign_error"; then
unset -f docker
rm -f "$calls" "$foreign_error"
task13_fail "cleanup accepted a transaction image with a foreign identity"
fi
if grep -Fq "image rm $foreign_ref" "$calls"; then
unset -f docker
rm -f "$calls" "$foreign_error"
task13_fail "cleanup attempted to remove a foreign transaction image"
fi
grep -Fq "refusing to remove foreign transaction image $foreign_ref" "$foreign_error" \
|| task13_fail "transaction image cleanup refusal was not explicit"
task13_remove_transaction_image "$owned_ref" 'sha256:owned'
[[ "$(grep -Fc "image rm $owned_ref" "$calls")" -eq 1 ]] \
|| task13_fail "cleanup did not remove exactly the owned transaction image reference"
unset -f docker
rm -f "$calls" "$foreign_error"
}
task13_self_test_image_evidence() (
local fixture records output
fixture="$(mktemp -d "${TMPDIR:-/tmp}/thothii-task15-image-evidence.XXXXXX")"
records="$fixture/records.tsv"
output="$fixture/images.json"
trap 'rm -rf "$fixture"' EXIT
: >"$records"
TASK13_RUN_ID="task15-image-run"
TASK13_SOURCE_COMMIT="0123456789abcdef0123456789abcdef01234567"
TASK13_IMAGE_EVIDENCE_STATUS="fail"
TASK13_IMAGE_EVIDENCE_RECORDS="$records"
TASK13_IMAGE_EVIDENCE_OUTPUT="$output"
TASK13_PROJECT="task15-image-project"
docker() {
case "$*" in
"container ls -aq --filter label=com.docker.compose.project=task15-image-project")
printf '%s\n' container-one container-two
;;
"container inspect --format {{.Image}} container-one")
printf '%s\n' 'sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'
;;
"container inspect --format {{.Image}} container-two")
printf '%s\n' 'sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb'
;;
"image inspect --format {{.Id}} fixture-candidate@sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc")
printf '%s\n' 'sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc'
;;
"image inspect --format {{.Id}} sha256:"*)
printf '%s\n' "${*: -1}"
;;
"image inspect --format {{json .RepoDigests}} sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
printf '%s\n' '["fixture/core@sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"]'
;;
"image inspect --format {{json .RepoDigests}} sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb")
printf '%s\n' '[]'
;;
"image inspect --format {{json .RepoDigests}} sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc")
printf '%s\n' '["fixture/candidate@sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"]'
;;
*) task13_fail "unexpected image evidence Docker command" ;;
esac
}
task13_record_image_evidence \
'fixture-candidate@sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc' \
'rollback-candidate'
task13_capture_failed_image_evidence
unset -f docker
node - "$output" <<'NODE'
const manifest = JSON.parse(require("node:fs").readFileSync(process.argv[2], "utf8"));
if (manifest.source_commit !== "0123456789abcdef0123456789abcdef01234567"
|| manifest.run_id !== "task15-image-run" || manifest.status !== "fail"
|| manifest.images.length !== 3) process.exit(1);
const ids = manifest.images.map((image) => image.id);
if (new Set(ids).size !== 3 || ids.some((id) => !/^sha256:[0-9a-f]{64}$/.test(id))) process.exit(1);
if (!manifest.images.some((image) => image.roles.includes("rollback-candidate")
&& image.repo_digests[0] === "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc")) process.exit(1);
NODE
)
task13_self_test_rollback_fixture_contract() {
[[ "${TASK13_BAD_CANDIDATE_BEHAVIOR:-}" == stopped ]] \
|| task13_fail "rollback candidate is not declared as guaranteed stopped"
[[ "$TASK13_BAD_CANDIDATE_IMAGE" =~ ^hello-world@sha256:[0-9a-f]{64}$ ]] \
|| task13_fail "rollback candidate is not the digest-pinned stopped fixture"
}
task13_self_test_runtime_binding_fixture() {
local root
root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)"
"$root/scripts/test-task13-runtime-fixtures.sh" local
}
task13_self_test_server_runtime_binding_fixture() {
local root
root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)"
"$root/scripts/test-task13-runtime-fixtures.sh" server
}
task13_self_test_stopped_project_containers() {
local calls foreign_error
calls="$(mktemp "${TMPDIR:-/tmp}/thothii-task13-project-containers.XXXXXX")"
foreign_error="$calls.foreign-error"
TASK13_PROJECT="thothii-0123456789ab"
TASK13_RUN_ID="task13-contract-run"
docker() {
printf '%s\n' "$*" >>"$calls"
case "$1 $2 $3" in
"container ls -aq") printf '%s\n' stopped-foreign ;;
"container inspect --format") printf '%s\n' some-other-run ;;
"volume ls -q"|"network ls -q") : ;;
*) return 1 ;;
esac
}
if task13_assert_project_ownership 2>"$foreign_error"; then
unset -f docker
rm -f "$calls" "$foreign_error"
task13_fail "project cleanup accepted a stopped foreign container"
fi
grep -Fq 'container ls -aq' "$calls" \
|| task13_fail "project cleanup did not enumerate stopped containers"
grep -Fq 'Compose project contains a foreign container resource' "$foreign_error" \
|| task13_fail "stopped foreign container refusal was not explicit"
: >"$calls"
docker() {
printf '%s\n' "$*" >>"$calls"
case "$1 $2 $3" in
"container ls -aq") printf '%s\n' stopped-owned ;;
"container inspect --format") printf '%s\n' "$TASK13_RUN_ID" ;;
"volume ls -q"|"network ls -q") : ;;
*) return 1 ;;
esac
}
task13_assert_project_ownership
[[ "$(grep -Fc 'container inspect --format' "$calls")" -eq 1 ]] \
|| task13_fail "project cleanup did not inspect exactly the stopped owned container"
unset -f docker
rm -f "$calls" "$foreign_error"
}
task13_self_test_timeout_process_group() {
local child_file child_pid="" rc
child_file="$(mktemp "${TMPDIR:-/tmp}/thothii-task13-timeout-child.XXXXXX")"
set +e
task13_bounded 1 "child-process regression" bash -c \
'sleep 30 & printf "%s\n" "$!" >"$1"; wait' _ "$child_file" >/dev/null 2>&1
rc=$?
set -e
child_pid="$(sed -n '1p' "$child_file")"
[[ "$rc" -ne 0 ]] || {
rm -f "$child_file"
task13_fail "timed command unexpectedly succeeded"
}
if [[ -n "$child_pid" ]] && kill -0 "$child_pid" 2>/dev/null; then
kill -KILL "$child_pid" 2>/dev/null || true
rm -f "$child_file"
task13_fail "timed command left its child process alive"
fi
rm -f "$child_file"
}
task13_self_test_nested_timeout_process_group() {
local child_file child_pid="" rc
child_file="$(mktemp "${TMPDIR:-/tmp}/thothii-task13-nested-timeout.XXXXXX")"
task13_nested_timeout_fixture() {
task13_bounded 30 "nested child-process regression" bash -c \
'sleep 30 & printf "%s\n" "$!" >"$1"; wait' _ "$child_file"
}
set +e
task13_supervise 1 "nested timeout supervisor" task13_nested_timeout_fixture >/dev/null 2>&1
rc=$?
set -e
unset -f task13_nested_timeout_fixture
child_pid="$(sed -n '1p' "$child_file")"
[[ "$rc" -ne 0 ]] || {
rm -f "$child_file"
task13_fail "nested timed command unexpectedly succeeded"
}
if [[ -n "$child_pid" ]] && kill -0 "$child_pid" 2>/dev/null; then
kill -KILL "$child_pid" 2>/dev/null || true
rm -f "$child_file"
task13_fail "outer timeout left its nested command child alive"
fi
rm -f "$child_file"
}
task13_self_test_public_timeout_contract() {
local root
root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)"
grep -Eq 'task13_supervise[[:space:]].*task13_smoke_main[[:space:]]+full' \
"$root/scripts/unified-deployment-smoke.sh" \
|| task13_fail "direct unified smoke invocation lacks an internal supervisor"
grep -Eq 'task13_supervise[[:space:]].*task13_smoke_main[[:space:]]+update' \
"$root/scripts/tht-update-smoke.sh" \
|| task13_fail "direct update smoke invocation lacks an internal supervisor"
grep -Eq 'task13_supervise[[:space:]].*task13_internal_semantic_smoke_main' \
"$root/scripts/internal-semantic-smoke.sh" \
|| task13_fail "direct internal semantic smoke invocation lacks an internal supervisor"
}
task13_self_test_internal_semantic_offline_contract() {
local root script
root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)"
script="$root/scripts/internal-semantic-smoke.sh"
grep -Fq 'task13_write_offline_semantic_override' "$script" \
|| task13_fail "internal semantic smoke lacks a dedicated offline override"
grep -Fq 'task13_offline_semantic_compose_logged "offline semantic recreation" \' "$script" \
|| task13_fail "offline semantic recreation must use the isolated offline compose wrapper"
grep -Fq 'up --detach --wait --wait-timeout 120 --pull never qdrant embedding' "$script" \
|| task13_fail "offline semantic recreation must start only qdrant and embedding with --pull never"
grep -Eq 'down --remove-orphans --timeout 10$' "$script" \
|| task13_fail "offline semantic phase must stop the stack before isolated recreation"
grep -Fq 'internal: true' "$script" \
|| task13_fail "offline semantic override must disable network egress"
if grep -Eq 'offline semantic recreation.*embedding-model-init|offline semantic recreation.*core|offline semantic recreation.*frontend' "$script"; then
task13_fail "offline semantic recreation must exclude bootstrap and non-semantic services"
fi
}
task13_self_test_windows_release_contract() {
local root script workflow
root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)"
script="$root/scripts/test-windows-clone-contract.ps1"
workflow="$root/.github/workflows/deployment.yml"
grep -Fq 'Task 13 path with spaces' "$script" \
|| task13_fail "Windows contract does not operate from a path containing spaces"
grep -Fq 'DockerStartup' "$script" \
|| task13_fail "Windows contract lacks an explicit Docker startup mode"
grep -Fq 'Kill($true)' "$script" \
|| task13_fail "Windows bounded runner does not kill the full process tree"
grep -Fq 'docker-desktop' "$workflow" \
|| task13_fail "workflow lacks a manual self-hosted Windows Docker Desktop gate"
grep -Fq -- '-DockerStartup' "$workflow" \
|| task13_fail "manual Windows release job does not execute Docker startup mode"
}
task13_self_test_server_release_contract() {
local root workflow server_smoke
root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)"
workflow="$root/.github/workflows/deployment.yml"
server_smoke="$root/scripts/server-deployment-smoke.sh"
[[ -x "$server_smoke" ]] || task13_fail "bounded Linux server deployment smoke is missing"
grep -Fq 'deploy/compose.server.yaml' "$root/scripts/unified-deployment-smoke.sh" \
|| task13_fail "server smoke does not load the server profile"
grep -Fq 'deploy/compose.session-server.yaml.example' "$root/scripts/unified-deployment-smoke.sh" \
|| task13_fail "server smoke does not load the required session overlay"
grep -Eq 'timeout .*scripts/server-deployment-smoke\.sh' "$workflow" \
|| task13_fail "workflow lacks an outer timeout for the Linux server smoke"
}
task13_self_test_registry_fingerprint() {
local fixture fixture_escaped before after linked
fixture="$(mktemp -d "${TMPDIR:-/tmp}/thothii-task13-registry-fingerprint.XXXXXX")"
printf -v fixture_escaped '%q' "$fixture"
trap "rm -rf -- $fixture_escaped; trap - RETURN" RETURN
before="$(task13_registry_filesystem_fingerprint "$fixture")"
[[ "$before" =~ ^sha256:[0-9a-f]{64}$ ]] \
|| task13_fail "registry fingerprint did not return a bounded digest"
[[ "$(task13_registry_filesystem_fingerprint "$fixture")" == "$before" ]] \
|| task13_fail "registry fingerprint changed without a filesystem mutation"
mkdir "$fixture/locks"
after="$(task13_registry_filesystem_fingerprint "$fixture")"
[[ "$after" != "$before" ]] || task13_fail "registry fingerprint ignored a new directory"
linked="$fixture/linked"
ln -s "$fixture/locks" "$linked" 2>/dev/null || return 0
if task13_registry_filesystem_fingerprint "$fixture" >/dev/null 2>&1; then
task13_fail "registry fingerprint accepted a symbolic link"
fi
}
task13_self_test_source_contract() {
local root host_network push_command registry_function workflow uses_count pinned_uses_count
local runner_preparation path
local auth_runtime_mount auth_root_mount auth_projection auth_runtime_owner
local pi_auth_bind pi_projection registry_runtime_mount registry_root_mount registry_projection
local application_secret_bind application_secret_mount application_secret_projection
local application_session_ca_fixture application_session_ca_source application_session_password_projection
local application_secret_parent_owner application_secret_file_owner
local image_evidence_environment image_evidence_initialization
local server_auth_projection_override server_auth_projection_descriptor
local server_auth_projection_environment server_auth_privileged_configure
local server_fixture_reclamation server_runtime_config_probe server_workspace_config_permission
local server_runtime_selector_probe server_runtime_generation_probe server_runtime_direct_file_probe
local server_runtime_projection_status_contract
local server_rollback_sentinel_read server_control_dir_reclamation
local server_checkpoint_lookup
local server_compose_privileged
local server_secret_source_owner server_secret_source_preparation server_tht_wrapper
root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)"
workflow="$root/.github/workflows/deployment.yml"
runner_preparation="$root/scripts/prepare-linux-docker-runner.sh"
host_network='--network'' host'
push_command='docker image ''push'
registry_function='task13_start_''registry'
auth_runtime_mount='auth-runtime:/run/thothii-''auth:ro'
auth_root_mount='$TASK13_AUTH_''ROOT:/run/thothii-auth:ro'
auth_projection='task13_prepare_local_auth_''runtime'
auth_runtime_owner='chown 10001:''10001 /target'
pi_auth_bind='$TASK13_PI_''AUTH:/home/thoth/.pi/agent/auth.json:ro'
pi_projection='task13_prepare_local_pi_''runtime'
registry_runtime_mount='registry-remote:/fixtures/remote.git:''ro'
registry_root_mount='$TASK13_''REMOTE:/fixtures/remote.git:ro'
registry_projection='task13_prepare_registry_''remote'
application_secret_bind='$TASK13_''SECRETS:/run/secrets/thothii.secrets:ro'
application_secret_mount='application-secrets:/run/''secrets:ro'
application_secret_projection='task13_prepare_local_application_''secrets'
application_session_ca_fixture='task13_write_session_ca_''fixture'
application_secret_parent_owner='chown 0:''0 /target'
application_session_ca_source='$TASK13_SESSION_''CA:/source/session_ca.pem:ro'
application_session_password_projection='cp /source/task13-runtime-password /target/session_''runtime_password'
application_secret_file_owner='chown 10001:''10001 /target/thothii.secrets /target/task13-runtime-password /target/session_runtime_password /target/session_ca.pem'
image_evidence_environment='TASK13_IMAGE_EVIDENCE_OUTPUT: ${{ runner.temp }}''/task13-images.json'
image_evidence_initialization='TASK13_IMAGE_EVIDENCE_OUTPUT="${TASK13_IMAGE_EVIDENCE_OUTPUT:-$TASK13_ROOT/''.artifacts/task-15/unified-docker-images.json}"'
server_auth_projection_override='deploy/compose.auth-runtime-''projection.yaml'
server_auth_projection_descriptor='runtime''Projection:'
server_auth_projection_environment='THT_AUTH_RUNTIME_''ROOT=%s'
server_auth_privileged_configure='sudo -n -- "$TASK13_''THT"'
server_fixture_reclamation='task13_reclaim_server_fixture_''ownership'
server_runtime_config_probe='task13_compose_''logged "verify server runtime configuration and secret readability"'
server_runtime_selector_probe='check_readable /run/thothii-auth/''CURRENT'
server_runtime_generation_probe='check_readable "/run/thothii-auth/generations/$projection_generation/''auth.yaml"'
server_runtime_direct_file_probe='check_readable /run/thothii-auth/''auth.yaml'
server_runtime_projection_status_contract='value.state!=="''ready"||value.equal!==true'
server_rollback_sentinel_read='sudo -n -- cat -- "$rollback_''sentinel"'
server_control_dir_reclamation='"$TASK13_TMP" "$TASK13_CONTROL_''DIR"'
server_checkpoint_lookup='task13_server_checkpoint_''leftover'
server_compose_privileged='sudo -n -- "$''@"'
server_secret_source_preparation='task13_prepare_server_secret_''sources'
server_secret_source_owner='chown 10001:''10001 -- "$TASK13_SECRETS" "$TASK13_PI_AUTH"'
server_tht_wrapper='task13_server_''tht'
server_workspace_config_permission='chmod 0644 "$TASK13_SERVER_''WORKSPACE_CONFIG"'
if rg -n 'docker[[:space:]]+(system[[:space:]]+)?prune' \
"$root/scripts/unified-deployment-smoke.sh" \
"$root/scripts/tht-update-smoke.sh" \
"$root/scripts/internal-semantic-smoke.sh" >/dev/null; then
task13_fail "Task 13 smoke scripts must never prune global Docker state"
fi
! grep -Fq -- "$host_network" "$root/scripts/unified-deployment-smoke.sh" \
|| task13_fail "the image registry must not depend on host networking"
if grep -Fq -- "$push_command" "$root/scripts/unified-deployment-smoke.sh" \
|| grep -Fq -- "$registry_function" "$root/scripts/unified-deployment-smoke.sh"; then
task13_fail "the rollback fixture must not depend on a daemon-to-host local image registry"
fi
grep -Fq -- "$auth_runtime_mount" "$root/scripts/unified-deployment-smoke.sh" \
|| task13_fail "the local smoke must mount a Compose-owned authentication runtime volume"
! grep -Fq -- "$auth_root_mount" "$root/scripts/unified-deployment-smoke.sh" \
|| task13_fail "the local smoke must not bind host-owned authentication into the core"
[[ "$(grep -Ec "^${auth_projection}\\(\\)|^[[:space:]]+${auth_projection}$" \
"$root/scripts/unified-deployment-smoke.sh")" -eq 2 ]] \
|| task13_fail "the local authentication runtime projection must be defined and invoked once"
grep -Fq -- "$auth_runtime_owner" "$root/scripts/unified-deployment-smoke.sh" \
|| task13_fail "the local authentication runtime projection must enforce the core UID"
! grep -Fq -- "$pi_auth_bind" "$root/scripts/unified-deployment-smoke.sh" \
|| task13_fail "the local smoke must not bind host-owned Pi authentication into the core"
[[ "$(grep -Ec "^${pi_projection}\\(\\)|^[[:space:]]+${pi_projection}$" \
"$root/scripts/unified-deployment-smoke.sh")" -eq 2 ]] \
|| task13_fail "the local Pi runtime projection must be defined and invoked once"
grep -Fq -- "$registry_runtime_mount" "$root/scripts/unified-deployment-smoke.sh" \
|| task13_fail "the local smoke must mount a Compose-owned Git fixture volume"
! grep -Fq -- "$registry_root_mount" "$root/scripts/unified-deployment-smoke.sh" \
|| task13_fail "the local smoke must not bind the host-owned Git fixture into the core"
[[ "$(grep -Ec "^${registry_projection}\\(\\)|^[[:space:]]+${registry_projection}$" \
"$root/scripts/unified-deployment-smoke.sh")" -ge 3 ]] \
|| task13_fail "the Git fixture projection must cover bootstrap and subsequent pushes"
! grep -Fq -- "$application_secret_bind" "$root/scripts/unified-deployment-smoke.sh" \
|| task13_fail "the local smoke must not bind the host-owned application bundle into the core"
grep -Fq -- "$application_secret_mount" "$root/scripts/unified-deployment-smoke.sh" \
|| task13_fail "the local smoke must mount Compose-owned application secrets"
[[ "$(grep -Ec "^${application_secret_projection}\\(\\)|^[[:space:]]+${application_secret_projection}$" \
"$root/scripts/unified-deployment-smoke.sh")" -eq 2 ]] \
|| task13_fail "the local application-secret projection must be defined and invoked once"
[[ "$(grep -Ec "^${application_session_ca_fixture}\\(\\)|^[[:space:]]+${application_session_ca_fixture}$" \
"$root/scripts/unified-deployment-smoke.sh")" -eq 3 ]] \
|| task13_fail "the session CA fixture must be defined and written for local and server smokes"
grep -Fq -- "$application_secret_parent_owner" "$root/scripts/unified-deployment-smoke.sh" \
|| task13_fail "the local application-secret mount root must remain root-owned"
grep -Fq -- "$application_session_ca_source" "$root/scripts/unified-deployment-smoke.sh" \
|| task13_fail "the local application-secret projection must include the session CA"
grep -Fq -- "$application_session_password_projection" "$root/scripts/unified-deployment-smoke.sh" \
|| task13_fail "the local application-secret projection must expose the session password name"
grep -Fq -- "$application_secret_file_owner" "$root/scripts/unified-deployment-smoke.sh" \
|| task13_fail "the projected application secrets must remain readable only by the core UID"
grep -Eq '^TASK13_BAD_CANDIDATE_IMAGE="[^"[:space:]]+@sha256:[0-9a-f]{64}"$' \
"$root/scripts/unified-deployment-smoke.sh" \
|| task13_fail "the bad rollback candidate must be an immutable digest reference"
grep -Eq 'timeout .*scripts/unified-deployment-smoke\.sh' "$workflow" \
|| task13_fail "CI lacks an outer timeout for the unified deployment smoke"
grep -Eq 'timeout .*scripts/tht-update-smoke\.sh' "$workflow" \
|| task13_fail "CI lacks an outer timeout for the tht update smoke"
grep -Fq 'bash scripts/prepare-linux-docker-runner.sh' "$workflow" \
|| task13_fail "CI must reclaim unused hosted-runner toolchains before the Docker release smoke"
[[ "$(grep -Fc -- "$image_evidence_environment" "$workflow")" -eq 3 ]] \
|| task13_fail "CI must write generated Docker image evidence outside the trusted checkout"
grep -Fq -- "$image_evidence_initialization" "$root/scripts/unified-deployment-smoke.sh" \
|| task13_fail "the Docker image evidence output must honor an explicit CI path"
grep -Fq -- "$server_auth_projection_override" "$root/scripts/unified-deployment-smoke.sh" \
|| task13_fail "the server smoke must mount the UID 10001 authentication projection"
grep -Fq -- "$server_auth_projection_descriptor" "$root/scripts/unified-deployment-smoke.sh" \
|| task13_fail "the server smoke installation must declare its authentication projection"
grep -Fq -- "$server_auth_projection_environment" "$root/scripts/unified-deployment-smoke.sh" \
|| task13_fail "the server smoke must export the authentication projection root"
grep -Fq -- "$server_auth_privileged_configure" "$root/scripts/unified-deployment-smoke.sh" \
|| task13_fail "the server smoke must publish projected authentication as root"
[[ "$(grep -Ec "^${server_fixture_reclamation}\\(\\)|^[[:space:]]+${server_fixture_reclamation}$" \
"$root/scripts/unified-deployment-smoke.sh")" -eq 2 ]] \
|| task13_fail "the server smoke must reclaim its root- and core-owned fixture exactly once"
grep -Fq -- "$server_runtime_config_probe" "$root/scripts/unified-deployment-smoke.sh" \
|| task13_fail "the server runtime preconditions must emit a named diagnostic"
grep -Fq -- "$server_runtime_selector_probe" "$root/scripts/unified-deployment-smoke.sh" \
|| task13_fail "the server runtime preconditions must verify the projection selector"
grep -Fq -- "$server_runtime_generation_probe" "$root/scripts/unified-deployment-smoke.sh" \
|| task13_fail "the server runtime preconditions must verify the selected generation"
! grep -Fq -- "$server_runtime_direct_file_probe" "$root/scripts/unified-deployment-smoke.sh" \
|| task13_fail "the server runtime preconditions must not require the forbidden direct-file fallback"
grep -Fq -- "$server_runtime_projection_status_contract" "$root/scripts/unified-deployment-smoke.sh" \
|| task13_fail "the server status assertion must validate projection readiness"
grep -Fq -- "$server_rollback_sentinel_read" "$root/scripts/unified-deployment-smoke.sh" \
|| task13_fail "the server rollback sentinel must be read through the privileged host surface"
grep -Fq -- "$server_control_dir_reclamation" "$root/scripts/unified-deployment-smoke.sh" \
|| task13_fail "server cleanup must reclaim both temporary and control roots"
[[ "$(grep -Ec "^${server_checkpoint_lookup}\\(\\)|${server_checkpoint_lookup}" \
"$root/scripts/unified-deployment-smoke.sh")" -eq 3 ]] \
|| task13_fail "server restore must inspect root-owned checkpoints through one privileged helper"
grep -Fq -- "$server_compose_privileged" "$root/scripts/unified-deployment-smoke.sh" \
|| task13_fail "server Compose operations must use the privileged host surface"
grep -Fq -- "$server_workspace_config_permission" "$root/scripts/unified-deployment-smoke.sh" \
|| task13_fail "the server workspace fixture must be readable by the container UID"
[[ "$(grep -Ec "^${server_secret_source_preparation}\\(\\)|^[[:space:]]+${server_secret_source_preparation}$" \
"$root/scripts/unified-deployment-smoke.sh")" -eq 2 ]] \
|| task13_fail "the server secret source preparation must be defined and invoked once"
grep -Fq -- "$server_secret_source_owner" "$root/scripts/unified-deployment-smoke.sh" \
|| task13_fail "the server secret sources must be private and owned by the container UID"
[[ "$(grep -Fc -- "$server_tht_wrapper" "$root/scripts/unified-deployment-smoke.sh")" -eq 10 ]] \
|| task13_fail "server operator commands must use the privileged canonical-auth wrapper"
[[ -x "$runner_preparation" ]] \
|| task13_fail "the Linux Docker runner preparation must be executable"
for path in /usr/local/lib/android /usr/share/dotnet /opt/ghc /usr/local/.ghcup; do
grep -Fq -- "$path" "$runner_preparation" \
|| task13_fail "the Linux Docker runner preparation is missing the scoped path: $path"
done
! grep -Eq '/opt/hostedtoolcache([/[:space:]]|$)|rm[[:space:]]+-rf[[:space:]]+--?[[:space:]]+/($|[[:space:]])' "$runner_preparation" \
|| task13_fail "the Linux Docker runner preparation must not remove required tool caches or broad roots"
uses_count="$(grep -Ec '^[[:space:]]+uses:' "$workflow")"
pinned_uses_count="$(grep -Ec '^[[:space:]]+uses: [^[:space:]]+@[0-9a-f]{40}([[:space:]]|$)' "$workflow")"
[[ "$uses_count" -gt 0 && "$uses_count" -eq "$pinned_uses_count" ]] \
|| task13_fail "every deployment workflow action must use a full immutable commit pin"
if grep -Fq '${{ secrets.' "$workflow"; then
task13_fail "the deployment release gate must not require repository secrets"
fi
for command in \
'bash scripts/verify-line-endings.sh' \
'bash scripts/test-unified-compose.sh' \
'bash scripts/test-compose-secret-policy.sh' \
'bash scripts/test-no-deployment-coupling.sh' \
'bash scripts/test-verify-workspace-install-docs.sh' \
'npx vitest run' \
'npx tsc --noEmit -p .' \
'npx tsc -b' \
'./scripts/test-windows-clone-contract.ps1'; do
grep -Fq "$command" "$workflow" || task13_fail "CI coverage is missing: $command"
done
}
task13_self_test() {
task13_self_test_sanitizer
task13_self_test_core_startup_diagnostics
task13_self_test_server_workspace_diagnostics
task13_self_test_cleanup_ownership
task13_self_test_image_cleanup_ownership
task13_self_test_transaction_image_cleanup
task13_self_test_image_evidence
task13_self_test_rollback_fixture_contract
task13_self_test_runtime_binding_fixture
task13_self_test_server_runtime_binding_fixture
task13_self_test_stopped_project_containers
task13_self_test_timeout_process_group
task13_self_test_nested_timeout_process_group
task13_self_test_public_timeout_contract
task13_self_test_internal_semantic_offline_contract
task13_self_test_windows_release_contract
task13_self_test_server_release_contract
task13_self_test_registry_fingerprint
task13_self_test_source_contract
printf 'Task 13 smoke safety contracts passed.\n'
}
task13_self_test_case() {
case "$1" in
rollback) task13_self_test_rollback_fixture_contract ;;
runtime-bindings) task13_self_test_runtime_binding_fixture ;;
server-bindings) task13_self_test_server_runtime_binding_fixture ;;
cleanup) task13_self_test_stopped_project_containers ;;
image-evidence) task13_self_test_image_evidence ;;
timeout-group) task13_self_test_timeout_process_group ;;
timeout-nested) task13_self_test_nested_timeout_process_group ;;
timeout-public) task13_self_test_public_timeout_contract ;;
semantic-offline) task13_self_test_internal_semantic_offline_contract ;;
windows) task13_self_test_windows_release_contract ;;
server) task13_self_test_server_release_contract ;;
server-diagnostics) task13_self_test_server_workspace_diagnostics ;;
startup-diagnostics) task13_self_test_core_startup_diagnostics ;;
*) task13_fail "unknown Task 13 self-test case: $1" ;;
esac
}
task13_fixtures_only() {
local tmp descriptor_path catalog_path initial_head updated_head
tmp="$(mktemp -d "${TMPDIR:-/tmp}/thothii-task13-fixtures.XXXXXX")"
trap 'rm -rf "$tmp"' RETURN
TASK13_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)"
TASK13_TMP="$tmp"
TASK13_REMOTE="$tmp/remote.git"
TASK13_SEED="$tmp/seed"
TASK13_BRANCH="task13-smoke"
TASK13_LOG="$tmp/task13.log"
TASK13_FAILURE_LOGGED=0
: >"$TASK13_LOG"
task13_seed_registry
descriptor_path="$TASK13_SEED/$TASK13_WORKSPACE_ID/workspace.yaml"
catalog_path="$TASK13_SEED/thoth-workspaces.yaml"
[[ -f "$catalog_path" ]] || task13_fail "missing Task 13 root workspace catalog"
[[ -f "$descriptor_path" ]] || task13_fail "missing Task 13 nested workspace descriptor"
[[ -f "$TASK13_SEED/$TASK13_WORKSPACE_ID/evidence/guide.md" ]] || task13_fail "missing Task 13 nested workspace evidence"
[[ ! -e "$TASK13_SEED/workspaces/$TASK13_WORKSPACE_ID.yaml" ]] || task13_fail "legacy Task 13 flat descriptor path exists"
[[ ! -e "$TASK13_SEED/workspace-content/$TASK13_WORKSPACE_ID" ]] || task13_fail "legacy Task 13 flat evidence path exists"
initial_head="$(git -C "$TASK13_SEED" rev-parse HEAD)"
task13_replace_once "$descriptor_path" 'name: Task 13 Smoke' 'name: Task 13 Smoke Updated'
task13_replace_once "$catalog_path" 'name: Task 13 Smoke' 'name: Task 13 Smoke Updated'
task13_commit_registry_change 'Update Task 13 workspace metadata'
updated_head="$(git -C "$TASK13_SEED" rev-parse HEAD)"
[[ "$updated_head" != "$initial_head" ]] || task13_fail "Task 13 metadata update did not advance Git head"
printf 'guide v2\n' >"$TASK13_SEED/$TASK13_WORKSPACE_ID/evidence/guide.md"
task13_commit_registry_change 'Update Task 13 workspace evidence only'
[[ "$(git -C "$TASK13_SEED" rev-parse HEAD)" != "$updated_head" ]] || task13_fail "Task 13 evidence-only commit did not advance Git head"
mkdir -p "$TASK13_SEED/workspaces" "$TASK13_SEED/workspace-content/$TASK13_WORKSPACE_ID/evidence"
cp "$TASK13_ROOT/scripts/fixtures/workspace-registry-task13.yaml" "$TASK13_SEED/workspaces/$TASK13_WORKSPACE_ID.yaml"
printf 'legacy guide\n' >"$TASK13_SEED/workspace-content/$TASK13_WORKSPACE_ID/evidence/guide.md"
[[ -f "$TASK13_SEED/workspaces/$TASK13_WORKSPACE_ID.yaml" ]] || task13_fail "missing Task 13 legacy flat descriptor fixture"
[[ -f "$TASK13_SEED/workspace-content/$TASK13_WORKSPACE_ID/evidence/guide.md" ]] || task13_fail "missing Task 13 legacy flat evidence fixture"
printf 'Task 13 fixture contract passed.
'
}
task13_initialize() {
local source_status
umask 077
TASK13_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)"
TASK13_TMP_PARENT="$(cd "${TMPDIR:-/tmp}" && pwd -P)"
TASK13_TMP="$(mktemp -d "$TASK13_TMP_PARENT/thothii-task13.XXXXXX")"
TASK13_TMP="$(cd "$TASK13_TMP" && pwd -P)"
TASK13_LOG="$TASK13_TMP/task13.log"
TASK13_FAILURE_LOGGED=0
: >"$TASK13_LOG"
trap 'task13_cleanup $?' EXIT
trap 'exit 130' INT TERM HUP
TASK13_RUN_ID="$(date -u +%Y%m%d%H%M%S)-$$-${RANDOM:-0}"
TASK13_SOURCE_COMMIT="$(git -C "$TASK13_ROOT" rev-parse --verify HEAD)"
[[ "$TASK13_SOURCE_COMMIT" =~ ^[0-9a-f]{40}$ ]] || task13_fail "source commit was not resolved"
source_status="$(git -C "$TASK13_ROOT" status --porcelain --untracked-files=normal \
| sed -e '/^?? \.playwright-cli\/$/d' -e '/^?? \.thothctl\/$/d')"
[[ -z "$source_status" ]] || task13_fail "source must be clean for image traceability"
TASK13_IMAGE_EVIDENCE_RECORDS="$TASK13_TMP/image-evidence.tsv"
: >"$TASK13_IMAGE_EVIDENCE_RECORDS"
TASK13_IMAGE_EVIDENCE_OUTPUT="${TASK13_IMAGE_EVIDENCE_OUTPUT:-$TASK13_ROOT/.artifacts/task-15/unified-docker-images.json}"
[[ "$TASK13_IMAGE_EVIDENCE_OUTPUT" = /* ]] \
|| task13_fail "Docker image evidence output must be an absolute path"
rm -f "$TASK13_IMAGE_EVIDENCE_OUTPUT"
TASK13_PROFILE="local"
TASK13_INSTALLATION="$TASK13_TMP/thothii-installation.yaml"
TASK13_PROJECT="thothii-$(task13_sha256_text "$TASK13_INSTALLATION" | cut -c1-12)"
TASK13_CONTROL_DIR="$TASK13_ROOT/.tht/$TASK13_PROJECT"
[[ ! -e "$TASK13_CONTROL_DIR" ]] || task13_fail "unique tht control directory already exists"
TASK13_CURRENT_IMAGE_OVERRIDE="$TASK13_CONTROL_DIR/current-image.yaml"
TASK13_UPDATE_STATE="$TASK13_CONTROL_DIR/update-state.json"
TASK13_REMOTE="$TASK13_TMP/remote.git"
TASK13_SEED="$TASK13_TMP/seed"
TASK13_BRANCH="task13-smoke"
TASK13_ENV_FILE="$TASK13_TMP/local.env"
TASK13_OVERRIDE="$TASK13_TMP/compose.task13.yaml"
TASK13_PI_AUTH="$TASK13_TMP/pi-auth.json"
TASK13_SECRETS="$TASK13_TMP/thothii.secrets"
TASK13_AUTH_ROOT="$TASK13_TMP/auth"
TASK13_AUTH_RUNTIME_ROOT="$TASK13_TMP/auth-runtime"
TASK13_AUTH_PASSWORD_FILE="$TASK13_TMP/local-auth-password"
TASK13_AUTH_RUNTIME_VOLUME="${TASK13_PROJECT}_auth-runtime"
TASK13_AUTH_PROJECTION_CONTAINER="$TASK13_PROJECT-auth-projection"
TASK13_PI_RUNTIME_VOLUME="${TASK13_PROJECT}_pi-state"
TASK13_PI_PROJECTION_CONTAINER="$TASK13_PROJECT-pi-projection"
TASK13_APPLICATION_SECRETS_VOLUME="${TASK13_PROJECT}_application-secrets"
TASK13_APPLICATION_SECRETS_PROJECTION_CONTAINER="$TASK13_PROJECT-application-secrets-projection"
TASK13_REGISTRY_RUNTIME_VOLUME="${TASK13_PROJECT}_registry-remote"
TASK13_REGISTRY_PROJECTION_CONTAINER="$TASK13_PROJECT-registry-projection"
TASK13_AUTH_ADMIN=task13-admin
TASK13_AUTH_PASSWORD="task13-auth-$TASK13_RUN_ID"
TASK13_OIDC_CLIENT_SECRET="task13-oidc-client-$TASK13_RUN_ID"
TASK13_AUTHENTIK_API_TOKEN="task13-authentik-token-$TASK13_RUN_ID"
TASK13_LLM_SERVER="$TASK13_TMP/fake-llm.mjs"
TASK13_OIDC_SERVER="$TASK13_TMP/fake-oidc.mjs"
TASK13_OIDC_CERT="$TASK13_TMP/fake-oidc-cert.pem"
TASK13_OIDC_KEY="$TASK13_TMP/fake-oidc-key.pem"
TASK13_THT_DIR="$TASK13_TMP/tht"
TASK13_LLM_CONTAINER="$TASK13_PROJECT-llm"
TASK13_OIDC_CONTAINER="$TASK13_PROJECT-oidc"
TASK13_BAD_CANDIDATE_CONTAINER="$TASK13_PROJECT-bad-candidate"
TASK13_CORE_IMAGE="task13-core-$TASK13_RUN_ID:local"
TASK13_FRONTEND_IMAGE="task13-frontend-$TASK13_RUN_ID:local"
TASK13_SECRET_VALUE="task13-secret-$TASK13_RUN_ID"
TASK13_NETWORK=""
TASK13_BAD_CANDIDATE_ID=""
TASK13_PREVIOUS_IMAGE_ID=""
TASK13_SERVER_DATA="$TASK13_TMP/Server Data"
TASK13_SERVER_PI_STATE="$TASK13_TMP/Server Pi State"
TASK13_SERVER_REGISTRY="$TASK13_TMP/Server Registry"
TASK13_SERVER_WORKSPACE_CONFIG="$TASK13_TMP/server-sessions.yaml"
TASK13_SESSION_RUNTIME_PASSWORD="$TASK13_TMP/session-runtime-password"
TASK13_SESSION_MIGRATOR_PASSWORD_FILE="$TASK13_TMP/session-migrator-password"
TASK13_SESSION_CA="$TASK13_TMP/session-ca.pem"
TASK13_SESSION_PASSWORD="task13-runtime-$TASK13_RUN_ID"
TASK13_SESSION_MIGRATOR_PASSWORD="task13-migrator-$TASK13_RUN_ID"
TASK13_FRONTEND_PORT=""
}
task13_require_tools() {
for command in bash git docker curl node openssl python3 sed awk grep rg sort; do
command -v "$command" >/dev/null 2>&1 || task13_fail "$command is required"
done
if [[ "${TASK13_PROFILE:-local}" == server ]]; then
command -v sudo >/dev/null 2>&1 || task13_fail "sudo is required for the Linux server smoke"
sudo -n -- true >/dev/null 2>&1 \
|| task13_fail "passwordless sudo is required for the Linux server smoke"
fi
task13_run_logged "Docker daemon readiness" docker info
task13_run_logged "Docker Compose readiness" docker compose version
task13_self_test_source_contract
}
task13_smoke_main() {
local mode="${1:-full}"
[[ "$mode" == full || "$mode" == update ]] || task13_fail "unknown Task 13 smoke mode: $mode"
task13_initialize
task13_require_tools
TASK13_FRONTEND_PORT="$(node -e 'const net=require("node:net"); const server=net.createServer(); server.listen(0,"127.0.0.1",()=>{process.stdout.write(String(server.address().port)); server.close()})')"
[[ "$TASK13_FRONTEND_PORT" =~ ^[1-9][0-9]*$ ]] || task13_fail "could not reserve a loopback frontend port"
task13_write_fixture_files
task13_write_environment /fixtures/remote.git
task13_seed_registry
task13_build_tht
task13_configure_local_authentication
task13_start_stack
task13_record_project_image_evidence
task13_record_image_evidence "$TASK13_CORE_IMAGE" fixture-runtime
task13_assert_project_ownership
task13_assert_built_image_ownership
task13_assert_runtime
task13_assert_local_restore_reauthentication
task13_prepare_persistence
if [[ "$mode" == full ]]; then
task13_registry_lifecycle
fi
task13_update_rollback
task13_record_project_image_evidence
TASK13_IMAGE_EVIDENCE_STATUS=pass
task13_write_image_evidence
printf 'Task 13 %s deployment smoke passed.\n' "$mode"
}
task13_server_smoke_main() {
task13_initialize
TASK13_PROFILE="server"
task13_require_tools
task13_write_server_fixture_files
task13_seed_registry
task13_build_tht
task13_prepare_server_auth_roots
task13_prepare_server_secret_sources
task13_configure_server_oidc_authentication
task13_start_server_stack
task13_record_project_image_evidence
task13_record_image_evidence "$TASK13_CORE_IMAGE" fixture-runtime
task13_assert_project_ownership
task13_assert_built_image_ownership
task13_assert_server_runtime
task13_assert_server_oidc_restore_verification
task13_record_project_image_evidence
TASK13_IMAGE_EVIDENCE_STATUS=pass
task13_write_image_evidence
printf 'Task 13 Linux server deployment smoke passed.\n'
}
if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
if [[ "${1:-}" == "--self-test" ]]; then
task13_self_test
elif [[ "${1:-}" == "--self-test-case" ]]; then
[[ -n "${2:-}" ]] || task13_fail "--self-test-case requires a case name"
task13_self_test_case "$2"
elif [[ "${1:-}" == "--fixtures-only" ]]; then
task13_fixtures_only
else
task13_supervise "$TASK13_SMOKE_TIMEOUT" "unified deployment smoke" task13_smoke_main full
fi
fi
Reference in New Issue View Git Blame Copy Permalink