fix(auth): address Task 13 deployment review findings
This commit is contained in:
@@ -24,8 +24,19 @@ if (services.join(",") !== "core,embedding,embedding-model-init,frontend,qdrant"
|
||||
if (/omics_portal|chirone|localllm_default|\/home\/chirone/i.test(JSON.stringify(config))) {
|
||||
throw new Error("default Compose contains application-specific coupling");
|
||||
}
|
||||
for (const volume of ["settings", "pi-state", "workspace-registry", "sessions", "auth-state", "qdrant-data", "embedding-models"]) {
|
||||
if (!config.volumes || !config.volumes[volume]) throw new Error(`missing required volume: ${volume}`);
|
||||
const expectedVolumes = [
|
||||
"auth-state",
|
||||
"embedding-models",
|
||||
"pi-state",
|
||||
"qdrant-data",
|
||||
"sessions",
|
||||
"settings",
|
||||
"workspace-registry",
|
||||
"workspace-secrets",
|
||||
];
|
||||
const actualVolumes = Object.keys(config.volumes || {});
|
||||
if (actualVolumes.join(",") !== expectedVolumes.join(",")) {
|
||||
throw new Error(`unexpected ordered volume set: ${actualVolumes.join(",")}`);
|
||||
}
|
||||
const core = config.services.core;
|
||||
const frontend = config.services.frontend;
|
||||
|
||||
@@ -166,6 +166,44 @@ task13_compose_logged() {
|
||||
task13_run_logged "$label" "${TASK13_COMPOSE[@]}" "$@"
|
||||
}
|
||||
|
||||
task13_report_core_startup_failure() {
|
||||
local container_id state exit_code logs cause="startup failure is unclassified"
|
||||
container_id="$(task13_compose ps --all -q core 2>/dev/null | head -n 1 || true)"
|
||||
state=""
|
||||
if [[ -n "$container_id" ]]; then
|
||||
state="$(docker inspect --format '{{.State.Status}}:{{.State.ExitCode}}' "$container_id" 2>/dev/null || true)"
|
||||
fi
|
||||
exit_code="${state##*:}"
|
||||
[[ "$exit_code" =~ ^[0-9]{1,3}$ ]] || exit_code="unknown"
|
||||
logs="$(task13_compose logs --no-color --tail 100 core 2>/dev/null || true)"
|
||||
case "$logs" in
|
||||
*auth_session_store_invalid*|*auth*storage*request*failed*|*EACCES*auth*|*permission*auth*)
|
||||
cause="authentication state storage is unavailable"
|
||||
;;
|
||||
*auth_config_invalid*|*authentication*configuration*)
|
||||
cause="authentication configuration is invalid"
|
||||
;;
|
||||
*workspace_registry_invalid*|*workspace*registry*)
|
||||
cause="workspace registry startup validation failed"
|
||||
;;
|
||||
esac
|
||||
printf 'Core startup cause: %s (exit code %s).\n' "$cause" "$exit_code" >&2
|
||||
}
|
||||
|
||||
task13_compose_start_logged() {
|
||||
local label="$1"
|
||||
shift
|
||||
task13_compose_files
|
||||
if task13_bounded "$TASK13_COMMAND_TIMEOUT" "$label" \
|
||||
"${TASK13_COMPOSE[@]}" "$@" >>"$TASK13_LOG" 2>&1; then
|
||||
return 0
|
||||
fi
|
||||
TASK13_FAILURE_LOGGED=1
|
||||
printf 'Task 13 command failed: %s\n' "$label" >&2
|
||||
task13_report_core_startup_failure
|
||||
return 1
|
||||
}
|
||||
|
||||
task13_write_environment() {
|
||||
local remote="$1"
|
||||
{
|
||||
@@ -296,6 +334,7 @@ services:
|
||||
- $TASK13_PI_MODELS:/home/thoth/.pi/agent/models.json:ro
|
||||
- $TASK13_PI_SETTINGS:/home/thoth/.pi/agent/settings.json:ro
|
||||
- workspace-registry:/data/workspace-registry
|
||||
- workspace-secrets:/data/workspace-secrets
|
||||
- sessions:/data/sessions
|
||||
- $TASK13_AUTH_ROOT:/run/thothii-auth:ro
|
||||
- auth-state:/data/auth
|
||||
@@ -333,6 +372,9 @@ volumes:
|
||||
workspace-registry:
|
||||
labels:
|
||||
io.thothii.task13.run: "$TASK13_RUN_ID"
|
||||
workspace-secrets:
|
||||
labels:
|
||||
io.thothii.task13.run: "$TASK13_RUN_ID"
|
||||
sessions:
|
||||
labels:
|
||||
io.thothii.task13.run: "$TASK13_RUN_ID"
|
||||
@@ -362,6 +404,9 @@ EOF
|
||||
|
||||
task13_write_server_fixture_files() {
|
||||
local data_root pi_root registry_root remote_path workspace_path
|
||||
TASK13_OIDC_SERVER="${TASK13_OIDC_SERVER:-$TASK13_TMP/fake-oidc.mjs}"
|
||||
TASK13_OIDC_CERT="${TASK13_OIDC_CERT:-$TASK13_TMP/fake-oidc-cert.pem}"
|
||||
TASK13_OIDC_KEY="${TASK13_OIDC_KEY:-$TASK13_TMP/fake-oidc-key.pem}"
|
||||
printf '{}\n' >"$TASK13_PI_AUTH"
|
||||
printf 'THT_MODEL_API_KEY=%s\nTHT_OIDC_CLIENT_SECRET=%s\nTHT_AUTHENTIK_API_TOKEN=%s\n' \
|
||||
"$TASK13_SECRET_VALUE" "$TASK13_OIDC_CLIENT_SECRET" "$TASK13_AUTHENTIK_API_TOKEN" >"$TASK13_SECRETS"
|
||||
@@ -376,6 +421,63 @@ EOF
|
||||
chmod 0600 "$TASK13_SECRETS" "$TASK13_SESSION_RUNTIME_PASSWORD" \
|
||||
"$TASK13_SESSION_MIGRATOR_PASSWORD_FILE"
|
||||
|
||||
task13_run_logged "create scoped fake OIDC certificate" openssl req -x509 -newkey rsa:2048 \
|
||||
-sha256 -nodes -days 1 -subj '/CN=task13-fake-oidc' \
|
||||
-addext 'subjectAltName=DNS:task13-fake-oidc' \
|
||||
-keyout "$TASK13_OIDC_KEY" -out "$TASK13_OIDC_CERT"
|
||||
chmod 0600 "$TASK13_OIDC_KEY"
|
||||
chmod 0644 "$TASK13_OIDC_CERT"
|
||||
cat >"$TASK13_OIDC_SERVER" <<'EOF'
|
||||
import { createPublicKey, generateKeyPairSync } from "node:crypto";
|
||||
import { readFileSync } from "node:fs";
|
||||
import https from "node:https";
|
||||
|
||||
const origin = "https://task13-fake-oidc:9443";
|
||||
const issuer = `${origin}/application/o/task13/`;
|
||||
const expectedToken = process.env.TASK13_AUTHENTIK_API_TOKEN;
|
||||
const { publicKey } = generateKeyPairSync("rsa", { modulusLength: 2048 });
|
||||
const jwk = { ...createPublicKey(publicKey).export({ format: "jwk" }), kid: "task13", use: "sig", alg: "RS256" };
|
||||
const send = (response, status, body) => {
|
||||
const payload = JSON.stringify(body);
|
||||
response.writeHead(status, { "content-type": "application/json", "content-length": Buffer.byteLength(payload) });
|
||||
response.end(payload);
|
||||
};
|
||||
|
||||
const server = https.createServer({
|
||||
cert: readFileSync("/fixtures/oidc-cert.pem"),
|
||||
key: readFileSync("/fixtures/oidc-key.pem"),
|
||||
}, (request, response) => {
|
||||
const target = new URL(request.url ?? "/", origin);
|
||||
if (target.pathname === "/health") return send(response, 200, { status: "ok" });
|
||||
if (target.pathname.includes(".well-known/openid-configuration")) {
|
||||
return send(response, 200, {
|
||||
issuer,
|
||||
authorization_endpoint: `${origin}/authorize`,
|
||||
token_endpoint: `${origin}/token`,
|
||||
jwks_uri: `${origin}/jwks`,
|
||||
response_types_supported: ["code"],
|
||||
subject_types_supported: ["public"],
|
||||
id_token_signing_alg_values_supported: ["RS256"],
|
||||
});
|
||||
}
|
||||
if (target.pathname === "/jwks") return send(response, 200, { keys: [jwk] });
|
||||
if (target.pathname === "/api/v3/core/groups/") {
|
||||
if (request.headers.authorization !== `Bearer ${expectedToken}`) return send(response, 401, { detail: "unauthorized" });
|
||||
const name = target.searchParams.get("name") ?? "";
|
||||
console.log(`group:${name}`);
|
||||
const configured = name === "task13-users" || name === "task13-admins";
|
||||
return send(response, 200, {
|
||||
pagination: { next: null },
|
||||
results: configured ? [{ name }, { name: "task13-unrelated" }] : [{ name: "task13-unrelated" }],
|
||||
});
|
||||
}
|
||||
return send(response, 404, { error: "not_found" });
|
||||
});
|
||||
|
||||
server.listen(9443, "0.0.0.0");
|
||||
EOF
|
||||
chmod 0644 "$TASK13_OIDC_SERVER"
|
||||
|
||||
cat >"$TASK13_SERVER_WORKSPACE_CONFIG" <<'EOF'
|
||||
language: en
|
||||
session_storage:
|
||||
@@ -417,6 +519,7 @@ services:
|
||||
labels:
|
||||
io.thothii.task13.run: "$TASK13_RUN_ID"
|
||||
environment:
|
||||
NODE_EXTRA_CA_CERTS: /fixtures/task13-oidc-ca.pem
|
||||
THT_WS_TASK13_SMOKE_DWH_TRANSPORT: postgres_direct
|
||||
THT_WS_TASK13_SMOKE_DWH_HOST: dwh.task13.invalid
|
||||
THT_WS_TASK13_SMOKE_DWH_PORT: "5432"
|
||||
@@ -427,6 +530,7 @@ services:
|
||||
volumes:
|
||||
- $remote_path:/fixtures/remote.git:ro
|
||||
- $TASK13_SESSION_RUNTIME_PASSWORD:/run/secrets/task13-runtime-password:ro
|
||||
- $TASK13_OIDC_CERT:/fixtures/task13-oidc-ca.pem:ro
|
||||
frontend:
|
||||
image: $TASK13_FRONTEND_IMAGE
|
||||
build:
|
||||
@@ -639,15 +743,15 @@ task13_configure_local_authentication() {
|
||||
task13_configure_server_oidc_authentication() {
|
||||
task13_run_logged "configure fake server OIDC authentication" "$TASK13_THT" --installation "$TASK13_INSTALLATION" auth configure \
|
||||
--mode oidc --public-url "https://task13.example.invalid" \
|
||||
--issuer "https://task13-fake-oidc.invalid/application/o/task13/" --client-id task13-smoke-client \
|
||||
--authentik-base-url "https://task13-fake-authentik.invalid" --user-group task13-users --admin-group task13-admins
|
||||
--issuer "https://task13-fake-oidc:9443/application/o/task13/" --client-id task13-smoke-client \
|
||||
--authentik-base-url "https://task13-fake-oidc:9443" --user-group task13-users --admin-group task13-admins
|
||||
}
|
||||
|
||||
task13_start_stack() {
|
||||
printf '== Build and start isolated local Compose distribution ==\n'
|
||||
task13_assert_rendered_contract
|
||||
task13_compose_logged "build local Compose images" build --pull
|
||||
task13_compose_logged "start local Compose distribution" up --detach --wait --wait-timeout 120
|
||||
task13_compose_start_logged "start local Compose distribution" up --detach --wait --wait-timeout 120
|
||||
TASK13_NETWORK="$(docker network ls \
|
||||
--filter "label=com.docker.compose.project=$TASK13_PROJECT" \
|
||||
--filter 'label=com.docker.compose.network=thothii' --format '{{.Name}}')"
|
||||
@@ -674,7 +778,36 @@ task13_start_server_stack() {
|
||||
printf '== Build and start isolated Linux server profile ==\n'
|
||||
task13_assert_rendered_contract
|
||||
task13_compose_logged "build server Compose images" build --pull core frontend
|
||||
task13_compose_logged "start server Compose distribution" \
|
||||
task13_compose_logged "create server Compose resources" create core frontend
|
||||
TASK13_NETWORK="$(docker network ls \
|
||||
--filter "label=com.docker.compose.project=$TASK13_PROJECT" \
|
||||
--filter 'label=com.docker.compose.network=thothii' --format '{{.Name}}')"
|
||||
[[ -n "$TASK13_NETWORK" && "$TASK13_NETWORK" != *$'\n'* ]] \
|
||||
|| task13_fail "isolated server Compose network was not resolved"
|
||||
task13_run_logged "start scoped fake OIDC provider" docker run --detach \
|
||||
--name "$TASK13_OIDC_CONTAINER" \
|
||||
--label "io.thothii.task13.run=$TASK13_RUN_ID" \
|
||||
--network "$TASK13_NETWORK" --network-alias task13-fake-oidc \
|
||||
--user "$(id -u):$(id -g)" \
|
||||
--env "TASK13_AUTHENTIK_API_TOKEN=$TASK13_AUTHENTIK_API_TOKEN" \
|
||||
--env NODE_EXTRA_CA_CERTS=/fixtures/oidc-cert.pem \
|
||||
--entrypoint node \
|
||||
--volume "$TASK13_OIDC_SERVER:/fixtures/fake-oidc.mjs:ro" \
|
||||
--volume "$TASK13_OIDC_CERT:/fixtures/oidc-cert.pem:ro" \
|
||||
--volume "$TASK13_OIDC_KEY:/fixtures/oidc-key.pem:ro" \
|
||||
"$TASK13_CORE_IMAGE" /fixtures/fake-oidc.mjs
|
||||
for _attempt in $(seq 1 30); do
|
||||
if docker exec "$TASK13_OIDC_CONTAINER" node -e \
|
||||
"fetch('https://task13-fake-oidc:9443/health',{signal:AbortSignal.timeout(3000)}).then(r=>{if(!r.ok)process.exit(1)}).catch(()=>process.exit(1))" \
|
||||
>>"$TASK13_LOG" 2>&1; then
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
docker exec "$TASK13_OIDC_CONTAINER" node -e \
|
||||
"fetch('https://task13-fake-oidc:9443/health',{signal:AbortSignal.timeout(3000)}).then(r=>{if(!r.ok)process.exit(1)}).catch(()=>process.exit(1))" \
|
||||
>>"$TASK13_LOG" 2>&1 || task13_log_failure "scoped fake OIDC provider readiness"
|
||||
task13_compose_start_logged "start server Compose distribution" \
|
||||
up --detach --wait --wait-timeout 120 core frontend
|
||||
}
|
||||
|
||||
@@ -729,6 +862,116 @@ task13_assert_local_auth_lifecycle() {
|
||||
[[ "$unauthenticated" == 401 ]] || task13_fail "local logout did not revoke the remembered session"
|
||||
}
|
||||
|
||||
task13_create_admin_session() {
|
||||
local frontend login_body me
|
||||
frontend="$(task13_frontend_address)"
|
||||
TASK13_ADMIN_COOKIE="$TASK13_TMP/operations-admin.cookies"
|
||||
login_body="$TASK13_TMP/operations-admin-login.json"
|
||||
printf '{"username":"%s","password":"%s","remember":true}' \
|
||||
"$TASK13_AUTH_ADMIN" "$TASK13_AUTH_PASSWORD" >"$login_body"
|
||||
chmod 0600 "$login_body"
|
||||
task13_run_logged "create authenticated smoke administration session" curl \
|
||||
--connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" --max-time "$TASK13_CURL_MAX_TIME" \
|
||||
--fail --silent --show-error --cookie-jar "$TASK13_ADMIN_COOKIE" \
|
||||
-H "Origin: http://$frontend" -H 'content-type: application/json' --data-binary "@$login_body" \
|
||||
"http://$frontend/api/auth/local/login"
|
||||
me="$(curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" --max-time "$TASK13_CURL_MAX_TIME" \
|
||||
--fail --silent --show-error --cookie "$TASK13_ADMIN_COOKIE" "http://$frontend/api/me")"
|
||||
TASK13_ADMIN_CSRF="$(node -e 'const value=JSON.parse(process.argv[1]); if(typeof value.csrfToken!=="string") process.exit(1); process.stdout.write(value.csrfToken)' "$me")" \
|
||||
|| task13_fail "authenticated smoke administration session lacks CSRF state"
|
||||
}
|
||||
|
||||
task13_authenticated_get() {
|
||||
local path="$1" frontend
|
||||
frontend="$(task13_frontend_address)"
|
||||
curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" --max-time "$TASK13_CURL_MAX_TIME" \
|
||||
--fail --silent --show-error --cookie "$TASK13_ADMIN_COOKIE" "http://$frontend/api/$path"
|
||||
}
|
||||
|
||||
task13_authenticated_post() {
|
||||
local path="$1" frontend
|
||||
frontend="$(task13_frontend_address)"
|
||||
curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" --max-time 15 \
|
||||
--fail --silent --show-error --cookie "$TASK13_ADMIN_COOKIE" \
|
||||
-H "Origin: http://$frontend" -H "x-thothii-csrf: $TASK13_ADMIN_CSRF" \
|
||||
-X POST "http://$frontend/api/$path"
|
||||
}
|
||||
|
||||
task13_assert_local_restore_reauthentication() {
|
||||
local frontend archive login_body cookie_before cookie_after me status_before status_after
|
||||
frontend="$(task13_frontend_address)"
|
||||
archive="$TASK13_TMP/local-restore-source.zip"
|
||||
login_body="$TASK13_TMP/local-restore-login.json"
|
||||
cookie_before="$TASK13_TMP/local-restore-before.cookies"
|
||||
cookie_after="$TASK13_TMP/local-restore-after.cookies"
|
||||
printf '{"username":"%s","password":"%s","remember":true}' \
|
||||
"$TASK13_AUTH_ADMIN" "$TASK13_AUTH_PASSWORD" >"$login_body"
|
||||
chmod 0600 "$login_body"
|
||||
|
||||
task13_run_logged "create pre-backup browser session" curl \
|
||||
--connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" --max-time "$TASK13_CURL_MAX_TIME" \
|
||||
--fail --silent --show-error --cookie-jar "$cookie_before" \
|
||||
-H "Origin: http://$frontend" -H 'content-type: application/json' --data-binary "@$login_body" \
|
||||
"http://$frontend/api/auth/local/login"
|
||||
task13_compose_logged "stop local stack for backup" stop
|
||||
task13_run_logged "create real default-custody backup" "$TASK13_THT" \
|
||||
--installation "$TASK13_INSTALLATION" backup --output "$archive"
|
||||
python3 - "$archive" <<'PY'
|
||||
import json
|
||||
import sys
|
||||
import zipfile
|
||||
|
||||
with zipfile.ZipFile(sys.argv[1]) as archive:
|
||||
manifest = json.loads(archive.read("manifest.json"))
|
||||
volumes = [item["logical_name"] for item in manifest["volumes"]]
|
||||
if volumes != ["embedding-models", "pi-state", "qdrant-data", "sessions", "settings", "workspace-registry", "workspace-secrets"]:
|
||||
raise SystemExit(f"unexpected backup volume custody: {volumes}")
|
||||
entries = manifest["entries"]
|
||||
if any(item.get("logical_name") == "auth-state" or "/data/auth" in item.get("source_path", "") for item in entries):
|
||||
raise SystemExit("default backup contains authentication runtime state")
|
||||
auth = [item for item in entries if item["path"].startswith("authentication-secrets/")]
|
||||
if len(auth) != 1 or not auth[0]["path"].endswith("-auth.yaml") or auth[0]["archived"]:
|
||||
raise SystemExit("default backup auth custody is not an auth.yaml reference only")
|
||||
if any(item["path"].endswith("users.yaml") for item in entries):
|
||||
raise SystemExit("default backup contains users.yaml")
|
||||
PY
|
||||
|
||||
task13_compose_start_logged "restart local stack before restore" up --detach --wait --wait-timeout 120
|
||||
status_before="$(curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" --max-time "$TASK13_CURL_MAX_TIME" \
|
||||
--silent --output /dev/null --write-out '%{http_code}' --cookie "$cookie_before" "http://$frontend/api/me")"
|
||||
[[ "$status_before" == 200 ]] || task13_fail "pre-backup browser session did not survive an ordinary stop/start"
|
||||
task13_run_logged "create post-backup browser session" curl \
|
||||
--connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" --max-time "$TASK13_CURL_MAX_TIME" \
|
||||
--fail --silent --show-error --cookie-jar "$cookie_after" \
|
||||
-H "Origin: http://$frontend" -H 'content-type: application/json' --data-binary "@$login_body" \
|
||||
"http://$frontend/api/auth/local/login"
|
||||
me="$(curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" --max-time "$TASK13_CURL_MAX_TIME" \
|
||||
--fail --silent --show-error --cookie "$cookie_after" "http://$frontend/api/me")"
|
||||
node -e 'const value=JSON.parse(process.argv[1]); if(value.issuer!=="local"||value.session?.remembered!==true) process.exit(1)' "$me" \
|
||||
|| task13_fail "post-backup browser session was not authenticated"
|
||||
task13_compose_logged "seed pending OIDC state excluded from restore" exec -T core sh -ceu \
|
||||
'printf %s "{}" > /data/auth/oidc/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.json && chmod 0600 /data/auth/oidc/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.json && test "$(find /data/auth/sessions -type f | wc -l | tr -d " ")" -ge 2'
|
||||
|
||||
task13_compose_logged "stop local stack for restore" stop
|
||||
task13_run_logged "perform real production restore" "$TASK13_THT" \
|
||||
--installation "$TASK13_INSTALLATION" restore "$archive" --yes
|
||||
task13_compose_start_logged "start restored local stack" up --detach --wait --wait-timeout 120
|
||||
status_before="$(curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" --max-time "$TASK13_CURL_MAX_TIME" \
|
||||
--silent --output /dev/null --write-out '%{http_code}' --cookie "$cookie_before" "http://$frontend/api/me")"
|
||||
status_after="$(curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" --max-time "$TASK13_CURL_MAX_TIME" \
|
||||
--silent --output /dev/null --write-out '%{http_code}' --cookie "$cookie_after" "http://$frontend/api/me")"
|
||||
[[ "$status_before" == 401 && "$status_after" == 401 ]] \
|
||||
|| task13_fail "restore did not force every independent browser session to reauthenticate"
|
||||
task13_compose_logged "verify private empty restored auth state" exec -T core sh -ceu '
|
||||
test "$(stat -c %a /data/auth)" = 700
|
||||
test "$(stat -c %a /data/auth/sessions)" = 700
|
||||
test "$(stat -c %a /data/auth/oidc)" = 700
|
||||
test "$(stat -c %u /data/auth)" = "$(id -u)"
|
||||
test -z "$(find /data/auth/sessions /data/auth/oidc -mindepth 1 -print -quit)"
|
||||
'
|
||||
task13_create_admin_session
|
||||
}
|
||||
|
||||
task13_assert_runtime() {
|
||||
local frontend expected_pi actual_pi core_id
|
||||
frontend="$(task13_frontend_address)"
|
||||
@@ -745,8 +988,10 @@ task13_assert_runtime() {
|
||||
'command -v pi >/dev/null'
|
||||
task13_compose_logged "core Docker socket isolation" exec -T core sh -ceu \
|
||||
'test ! -e /var/run/docker.sock'
|
||||
task13_compose_logged "workspace registry bootstrap" exec -T core \
|
||||
curl --connect-timeout 3 --max-time 10 -fsS http://127.0.0.1:8787/workspace-registry/status
|
||||
task13_assert_local_auth_lifecycle
|
||||
task13_create_admin_session
|
||||
task13_authenticated_get workspace-registry/status >/dev/null \
|
||||
|| task13_fail "authenticated workspace registry bootstrap failed"
|
||||
task13_compose_logged "active workspace registry state" exec -T core sh -ceu \
|
||||
'test -f /data/workspace-registry/state/active.json'
|
||||
task13_compose_logged "mounted Pi auth readability" exec -T core sh -ceu \
|
||||
@@ -757,7 +1002,6 @@ task13_assert_runtime() {
|
||||
[[ "$(docker inspect --format '{{ index .Config.Labels "io.thothii.task13.run" }}' "$core_id")" == "$TASK13_RUN_ID" ]] \
|
||||
|| task13_fail "core lacks the explicit Task 13 resource label"
|
||||
task13_assert_maintenance_auth_isolation
|
||||
task13_assert_local_auth_lifecycle
|
||||
task13_run_logged "tht Pi doctor" "$TASK13_THT" --installation "$TASK13_INSTALLATION" pi doctor
|
||||
}
|
||||
|
||||
@@ -799,7 +1043,7 @@ task13_report_server_workspace_failure() {
|
||||
}
|
||||
|
||||
task13_assert_server_runtime() {
|
||||
local frontend unauthenticated trusted_header_status session_status diagnostics diagnostic_status core_id frontend_id
|
||||
local frontend unauthenticated trusted_header_status session_status diagnostics status provider_requests core_id frontend_id
|
||||
local expected_core_image expected_frontend_image
|
||||
frontend="$(task13_frontend_address)"
|
||||
task13_run_logged "server frontend health" curl \
|
||||
@@ -857,22 +1101,32 @@ task13_assert_server_runtime() {
|
||||
if grep -Fq "$TASK13_SECRET_VALUE" "$TASK13_TMP/server-sessions.out"; then
|
||||
task13_fail "server session failure exposed the fixture secret"
|
||||
fi
|
||||
status="$TASK13_TMP/server-auth-status.json"
|
||||
task13_run_logged "server static OIDC status" "$TASK13_THT" --installation "$TASK13_INSTALLATION" auth status --json
|
||||
"$TASK13_THT" --installation "$TASK13_INSTALLATION" auth status --json >"$status"
|
||||
node -e 'const value=JSON.parse(require("fs").readFileSync(process.argv[1], "utf8")); if(value.mode!=="oidc"||typeof value.configRevision!=="string"||value.configRevision.length!==64) process.exit(1)' "$status" \
|
||||
|| task13_fail "server static OIDC status was not valid"
|
||||
diagnostics="$TASK13_TMP/server-auth-diagnostics.json"
|
||||
set +e
|
||||
"$TASK13_THT" --installation "$TASK13_INSTALLATION" auth check --json >"$diagnostics" 2>>"$TASK13_LOG"
|
||||
diagnostic_status=$?
|
||||
set -e
|
||||
[[ "$diagnostic_status" == 1 ]] || task13_fail "fake OIDC diagnostics did not fail closed"
|
||||
node -e 'const value=JSON.parse(require("fs").readFileSync(process.argv[1], "utf8")); if(value.mode!=="oidc"||value.ready!==false||!value.checks.some((item)=>item.code==="oidc_discovery_unreachable")) process.exit(1)' "$diagnostics" \
|
||||
|| task13_fail "fake OIDC fixture did not produce the expected static diagnostic"
|
||||
task13_run_logged "server live OIDC diagnostics" "$TASK13_THT" --installation "$TASK13_INSTALLATION" auth check --json
|
||||
"$TASK13_THT" --installation "$TASK13_INSTALLATION" auth check --json >"$diagnostics"
|
||||
node -e 'const value=JSON.parse(require("fs").readFileSync(process.argv[1], "utf8")); if(value.mode!=="oidc"||value.ready!==true||value.checks.length!==1||value.checks[0].code!=="auth_ready") process.exit(1)' "$diagnostics" \
|
||||
|| task13_fail "scoped fake OIDC provider did not pass live production diagnostics"
|
||||
provider_requests="$(docker logs "$TASK13_OIDC_CONTAINER" 2>>"$TASK13_LOG")"
|
||||
[[ "$(grep -Fc 'group:task13-users' <<<"$provider_requests")" -ge 1 ]] \
|
||||
|| task13_fail "live OIDC diagnostics did not verify the mandatory user group"
|
||||
[[ "$(grep -Fc 'group:task13-admins' <<<"$provider_requests")" -ge 1 ]] \
|
||||
|| task13_fail "live OIDC diagnostics did not verify the mandatory administrator group"
|
||||
if grep -Fq 'group:task13-unrelated' <<<"$provider_requests" \
|
||||
|| grep -Fq 'task13-unrelated' "$diagnostics"; then
|
||||
task13_fail "live OIDC diagnostics queried or warned about an unrelated group"
|
||||
fi
|
||||
if grep -Fq "$TASK13_OIDC_CLIENT_SECRET" "$diagnostics" || grep -Fq "$TASK13_AUTHENTIK_API_TOKEN" "$diagnostics"; then
|
||||
task13_fail "OIDC diagnostics exposed a fixture secret"
|
||||
fi
|
||||
}
|
||||
|
||||
task13_registry_status() {
|
||||
task13_compose exec -T core curl --connect-timeout 3 --max-time 10 -fsS \
|
||||
http://127.0.0.1:8787/workspace-registry/status
|
||||
task13_authenticated_get workspace-registry/status
|
||||
}
|
||||
|
||||
task13_registry_head() {
|
||||
@@ -914,8 +1168,7 @@ task13_prepare_persistence() {
|
||||
TASK13_INITIAL_MOUNTS="$(task13_mount_fingerprint)"
|
||||
TASK13_INITIAL_HEAD="$(task13_active_registry_head)"
|
||||
[[ "$TASK13_INITIAL_HEAD" =~ ^[0-9a-f]{40}$ ]] || task13_fail "initial registry head is invalid"
|
||||
task13_compose exec -T core curl --connect-timeout 3 --max-time 10 -fsS \
|
||||
http://127.0.0.1:8787/workspaces \
|
||||
task13_authenticated_get workspaces \
|
||||
| grep -Fq 'Task 13 Smoke' || task13_fail "initial workspace is unavailable"
|
||||
}
|
||||
|
||||
@@ -939,8 +1192,8 @@ task13_registry_lifecycle() {
|
||||
task13_commit_registry_change 'Update Task 13 workspace metadata'
|
||||
task13_write_environment /fixtures/remote.git
|
||||
task13_compose_logged "online Compose recreation" up --detach --force-recreate --wait --wait-timeout 120
|
||||
task13_compose exec -T core curl --connect-timeout 3 --max-time 15 -fsS -X POST http://127.0.0.1:8787/workspace-registry/pull >/dev/null
|
||||
task13_compose exec -T core curl --connect-timeout 3 --max-time 10 -fsS http://127.0.0.1:8787/workspaces | grep -Fq 'Task 13 Smoke Updated' || task13_fail "valid Git update was not activated"
|
||||
task13_authenticated_post workspace-registry/pull >/dev/null
|
||||
task13_authenticated_get workspaces | grep -Fq 'Task 13 Smoke Updated' || task13_fail "valid Git update was not activated"
|
||||
valid_head="$(task13_active_registry_head)"
|
||||
[[ "$valid_head" =~ ^[0-9a-f]{40}$ && "$valid_head" != "$TASK13_INITIAL_HEAD" ]] || task13_fail "valid Git update did not advance the registry head"
|
||||
TASK13_INITIAL_HEAD="$valid_head"
|
||||
@@ -950,7 +1203,7 @@ task13_registry_lifecycle() {
|
||||
'
|
||||
printf 'guide v2\n' >"$TASK13_SEED/$TASK13_WORKSPACE_ID/evidence/guide.md"
|
||||
task13_commit_registry_change 'Update Task 13 workspace evidence only'
|
||||
task13_compose exec -T core curl --connect-timeout 3 --max-time 15 -fsS -X POST http://127.0.0.1:8787/workspace-registry/pull >/dev/null
|
||||
task13_authenticated_post workspace-registry/pull >/dev/null
|
||||
evidence_head="$(task13_active_registry_head)"
|
||||
[[ "$evidence_head" =~ ^[0-9a-f]{40}$ && "$evidence_head" != "$valid_head" ]] || task13_fail "content-only Git Evidence update did not advance the registry head"
|
||||
TASK13_INITIAL_HEAD="$evidence_head"
|
||||
@@ -960,14 +1213,14 @@ task13_registry_lifecycle() {
|
||||
'
|
||||
task13_replace_once "$TASK13_SEED/thoth-workspaces.yaml" 'name: Task 13 Smoke Updated' 'name: Task 13 Smoke Drift'
|
||||
task13_commit_registry_change 'Break Task 13 workspace metadata parity'
|
||||
if task13_compose exec -T core curl --connect-timeout 3 --max-time 15 -fsS -X POST http://127.0.0.1:8787/workspace-registry/pull >>"$TASK13_LOG" 2>&1; then
|
||||
if task13_authenticated_post workspace-registry/pull >>"$TASK13_LOG" 2>&1; then
|
||||
task13_fail "registry accepted catalog/descriptor metadata mismatch"
|
||||
fi
|
||||
[[ "$(task13_active_registry_head)" == "$TASK13_INITIAL_HEAD" ]] || task13_fail "metadata mismatch replaced the valid registry head"
|
||||
task13_compose exec -T core curl --connect-timeout 3 --max-time 10 -fsS http://127.0.0.1:8787/workspaces | grep -Fq 'Task 13 Smoke Updated' || task13_fail "metadata mismatch displaced the valid workspace"
|
||||
task13_authenticated_get workspaces | grep -Fq 'Task 13 Smoke Updated' || task13_fail "metadata mismatch displaced the valid workspace"
|
||||
task13_replace_once "$TASK13_SEED/thoth-workspaces.yaml" 'name: Task 13 Smoke Drift' 'name: Task 13 Smoke Updated'
|
||||
task13_commit_registry_change 'Restore Task 13 workspace metadata parity'
|
||||
task13_compose exec -T core curl --connect-timeout 3 --max-time 15 -fsS -X POST http://127.0.0.1:8787/workspace-registry/pull >/dev/null
|
||||
task13_authenticated_post workspace-registry/pull >/dev/null
|
||||
repaired_head="$(task13_active_registry_head)"
|
||||
[[ "$repaired_head" =~ ^[0-9a-f]{40}$ && "$repaired_head" != "$TASK13_INITIAL_HEAD" ]] || task13_fail "metadata repair did not restore a fresh valid registry head"
|
||||
TASK13_INITIAL_HEAD="$repaired_head"
|
||||
@@ -983,14 +1236,14 @@ task13_registry_lifecycle() {
|
||||
mkdir -p "$TASK13_SEED/orphan/evidence"
|
||||
printf 'orphan guide\n' >"$TASK13_SEED/orphan/evidence/guide.md"
|
||||
task13_commit_registry_change 'Add orphan Task 13 workspace directory'
|
||||
if task13_compose exec -T core curl --connect-timeout 3 --max-time 15 -fsS -X POST http://127.0.0.1:8787/workspace-registry/pull >>"$TASK13_LOG" 2>&1; then
|
||||
if task13_authenticated_post workspace-registry/pull >>"$TASK13_LOG" 2>&1; then
|
||||
task13_fail "registry accepted orphan Task 13 descriptor directory"
|
||||
fi
|
||||
[[ "$(task13_active_registry_head)" == "$TASK13_INITIAL_HEAD" ]] || task13_fail "orphan descriptor directory replaced the valid registry head"
|
||||
task13_compose exec -T core curl --connect-timeout 3 --max-time 10 -fsS http://127.0.0.1:8787/workspaces | grep -Fq 'Task 13 Smoke Updated' || task13_fail "orphan descriptor displaced the valid workspace"
|
||||
task13_authenticated_get workspaces | grep -Fq 'Task 13 Smoke Updated' || task13_fail "orphan descriptor displaced the valid workspace"
|
||||
rm -rf "$TASK13_SEED/orphan"
|
||||
task13_commit_registry_change 'Remove orphan Task 13 workspace directory'
|
||||
task13_compose exec -T core curl --connect-timeout 3 --max-time 15 -fsS -X POST http://127.0.0.1:8787/workspace-registry/pull >/dev/null
|
||||
task13_authenticated_post workspace-registry/pull >/dev/null
|
||||
TASK13_INITIAL_HEAD="$(task13_active_registry_head)"
|
||||
|
||||
printf '== Reject the retired flat workspace layout and retain the valid snapshot ==
|
||||
@@ -999,11 +1252,11 @@ task13_registry_lifecycle() {
|
||||
cp "$TASK13_ROOT/scripts/fixtures/workspace-registry-task13.yaml" "$TASK13_SEED/workspaces/$TASK13_WORKSPACE_ID.yaml"
|
||||
printf 'legacy guide\n' >"$TASK13_SEED/workspace-content/$TASK13_WORKSPACE_ID/evidence/guide.md"
|
||||
task13_commit_registry_change 'Reintroduce retired flat Task 13 workspace layout'
|
||||
if task13_compose exec -T core curl --connect-timeout 3 --max-time 15 -fsS -X POST http://127.0.0.1:8787/workspace-registry/pull >>"$TASK13_LOG" 2>&1; then
|
||||
if task13_authenticated_post workspace-registry/pull >>"$TASK13_LOG" 2>&1; then
|
||||
task13_fail "registry accepted the retired flat Task 13 workspace layout"
|
||||
fi
|
||||
[[ "$(task13_active_registry_head)" == "$TASK13_INITIAL_HEAD" ]] || task13_fail "retired flat workspace layout replaced the valid registry head"
|
||||
task13_compose exec -T core curl --connect-timeout 3 --max-time 10 -fsS http://127.0.0.1:8787/workspaces | grep -Fq 'Task 13 Smoke Updated' || task13_fail "retired flat workspace layout displaced the valid workspace"
|
||||
task13_authenticated_get workspaces | grep -Fq 'Task 13 Smoke Updated' || task13_fail "retired flat workspace layout displaced the valid workspace"
|
||||
task13_assert_sentinels
|
||||
}
|
||||
|
||||
@@ -1059,8 +1312,7 @@ task13_update_rollback() {
|
||||
task13_assert_sentinels
|
||||
task13_run_logged "post-rollback tht doctor" \
|
||||
"$TASK13_THT" --installation "$TASK13_INSTALLATION" pi doctor
|
||||
task13_compose exec -T core curl --connect-timeout 3 --max-time 10 -fsS \
|
||||
http://127.0.0.1:8787/workspaces \
|
||||
task13_authenticated_get workspaces \
|
||||
| grep -Fq 'Task 13 Smoke' || task13_fail "rollback lost the active workspace"
|
||||
}
|
||||
|
||||
@@ -1174,6 +1426,7 @@ task13_cleanup() {
|
||||
fi
|
||||
task13_remove_labeled_container "${TASK13_BAD_CANDIDATE_CONTAINER:-}" || cleanup_rc=1
|
||||
task13_remove_labeled_container "${TASK13_LLM_CONTAINER:-}" || cleanup_rc=1
|
||||
task13_remove_labeled_container "${TASK13_OIDC_CONTAINER:-}" || cleanup_rc=1
|
||||
if [[ -n "${TASK13_PROJECT:-}" && -n "${TASK13_ROOT:-}" && -f "${TASK13_OVERRIDE:-}" ]]; then
|
||||
if task13_assert_project_ownership >>"${TASK13_LOG:-/dev/null}" 2>&1; then
|
||||
task13_compose_files
|
||||
@@ -1304,6 +1557,40 @@ task13_self_test_server_workspace_diagnostics() {
|
||||
|| task13_fail "server diagnostics did not sanitize response and core logs"
|
||||
}
|
||||
|
||||
task13_self_test_core_startup_diagnostics() {
|
||||
local output
|
||||
TASK13_SECRET_VALUE="fixture-known-secret"
|
||||
|
||||
task13_compose() {
|
||||
case "$*" in
|
||||
"ps --all -q core") printf '%s\n' 'task13-core-id' ;;
|
||||
"logs --no-color --tail 100 core")
|
||||
printf '%s\n' \
|
||||
'Error: EACCES: permission denied, mkdir /data/auth/sessions' \
|
||||
'password=plain-secret token=fixture-known-secret' \
|
||||
'secret path: /run/secrets/private-token' \
|
||||
' at createFileAuthSessionStore (/app/backend/dist/auth/session-store.js:101:9)'
|
||||
;;
|
||||
*) task13_fail "startup diagnostics requested an unexpected Compose command: $*" ;;
|
||||
esac
|
||||
}
|
||||
docker() {
|
||||
[[ "$*" == "inspect --format {{.State.Status}}:{{.State.ExitCode}} task13-core-id" ]] \
|
||||
|| task13_fail "startup diagnostics requested an unexpected Docker command: $*"
|
||||
printf '%s\n' 'exited:1'
|
||||
}
|
||||
|
||||
output="$(task13_report_core_startup_failure 2>&1)"
|
||||
unset -f task13_compose docker
|
||||
|
||||
[[ "$output" == 'Core startup cause: authentication state storage is unavailable (exit code 1).' ]] \
|
||||
|| task13_fail "startup diagnostics emitted a non-allowlisted cause: $output"
|
||||
for leaked in EACCES permission /data/auth /run/secrets plain-secret fixture-known-secret \
|
||||
createFileAuthSessionStore session-store.js; do
|
||||
[[ "$output" != *"$leaked"* ]] || task13_fail "startup diagnostics leaked $leaked"
|
||||
done
|
||||
}
|
||||
|
||||
task13_self_test_cleanup_ownership() {
|
||||
local calls foreign_error owned_name foreign_name
|
||||
calls="$(mktemp "${TMPDIR:-/tmp}/thothii-task13-cleanup-contract.XXXXXX")"
|
||||
@@ -1664,6 +1951,7 @@ task13_self_test_source_contract() {
|
||||
|
||||
task13_self_test() {
|
||||
task13_self_test_sanitizer
|
||||
task13_self_test_core_startup_diagnostics
|
||||
task13_self_test_server_workspace_diagnostics
|
||||
task13_self_test_cleanup_ownership
|
||||
task13_self_test_image_cleanup_ownership
|
||||
@@ -1695,6 +1983,7 @@ task13_self_test_case() {
|
||||
windows) task13_self_test_windows_release_contract ;;
|
||||
server) task13_self_test_server_release_contract ;;
|
||||
server-diagnostics) task13_self_test_server_workspace_diagnostics ;;
|
||||
startup-diagnostics) task13_self_test_core_startup_diagnostics ;;
|
||||
*) task13_fail "unknown Task 13 self-test case: $1" ;;
|
||||
esac
|
||||
}
|
||||
@@ -1777,8 +2066,12 @@ task13_initialize() {
|
||||
TASK13_PI_MODELS="$TASK13_TMP/models.json"
|
||||
TASK13_PI_SETTINGS="$TASK13_TMP/pi-settings.json"
|
||||
TASK13_LLM_SERVER="$TASK13_TMP/fake-llm.mjs"
|
||||
TASK13_OIDC_SERVER="$TASK13_TMP/fake-oidc.mjs"
|
||||
TASK13_OIDC_CERT="$TASK13_TMP/fake-oidc-cert.pem"
|
||||
TASK13_OIDC_KEY="$TASK13_TMP/fake-oidc-key.pem"
|
||||
TASK13_THT_DIR="$TASK13_TMP/tht"
|
||||
TASK13_LLM_CONTAINER="$TASK13_PROJECT-llm"
|
||||
TASK13_OIDC_CONTAINER="$TASK13_PROJECT-oidc"
|
||||
TASK13_BAD_CANDIDATE_CONTAINER="$TASK13_PROJECT-bad-candidate"
|
||||
TASK13_CORE_IMAGE="task13-core-$TASK13_RUN_ID:local"
|
||||
TASK13_FRONTEND_IMAGE="task13-frontend-$TASK13_RUN_ID:local"
|
||||
@@ -1799,7 +2092,7 @@ task13_initialize() {
|
||||
}
|
||||
|
||||
task13_require_tools() {
|
||||
for command in bash git docker curl node sed awk grep rg sort; do
|
||||
for command in bash git docker curl node openssl python3 sed awk grep rg sort; do
|
||||
command -v "$command" >/dev/null 2>&1 || task13_fail "$command is required"
|
||||
done
|
||||
task13_run_logged "Docker daemon readiness" docker info
|
||||
@@ -1823,6 +2116,7 @@ task13_smoke_main() {
|
||||
task13_assert_project_ownership
|
||||
task13_assert_built_image_ownership
|
||||
task13_assert_runtime
|
||||
task13_assert_local_restore_reauthentication
|
||||
task13_prepare_persistence
|
||||
if [[ "$mode" == full ]]; then
|
||||
task13_registry_lifecycle
|
||||
|
||||
Reference in New Issue
Block a user