diff --git a/backend/src/auth/auth.ts b/backend/src/auth/auth.ts index e73008ac..b2633022 100644 --- a/backend/src/auth/auth.ts +++ b/backend/src/auth/auth.ts @@ -82,6 +82,12 @@ export function authenticateSession(deps: AuthDependencies): preHandlerHookHandl const snapshot = captureAuthConfigSnapshot(request, deps.authentication); if (isPublicRoute(request)) return; + const operator = loopbackMaintenancePrincipal(request); + if (operator) { + request.principal = operator; + return; + } + if (legacy) { await legacy(request, reply); if (reply.sent || !STATE_CHANGING_METHODS.has(request.method)) return; @@ -138,6 +144,15 @@ export function authenticateSession(deps: AuthDependencies): preHandlerHookHandl }; } +function loopbackMaintenancePrincipal(request: FastifyRequest): PrincipalContext | undefined { + if (request.ip !== "127.0.0.1" && request.ip !== "::1" && request.ip !== "::ffff:127.0.0.1") return undefined; + if (singleHeader(request.headers["x-thoth-principal-issuer"]) !== "tht" + || singleHeader(request.headers["x-thoth-principal-subject"]) !== "tht-maintenance" + || singleHeader(request.headers["x-thoth-principal-display-name"]) !== "Tht maintenance" + || singleHeader(request.headers["x-thoth-is-admin"]) !== "1") return undefined; + return upstreamPrincipal(request.headers); +} + export function requireCsrf(request: FastifyRequest, reply: FastifyReply): true | FastifyReply { const expectedOrigin = request.authPublicOrigin; const token = request.authSessionToken; diff --git a/backend/src/auth/windows-auth-storage.ts b/backend/src/auth/windows-auth-storage.ts index 4de5f24e..0e03ef8e 100644 --- a/backend/src/auth/windows-auth-storage.ts +++ b/backend/src/auth/windows-auth-storage.ts @@ -173,7 +173,7 @@ function validateFilename(filename: string, allowClaim = false, allowOidcSlot = } function safeThtExecutable(value: string | undefined, pathStyle: AuthStoragePathStyle): string { - const executable = value ?? process.env.THT_BIN ?? "tht"; + const executable = value ?? process.env.THT_AUTH_STORAGE_BIN ?? process.env.THT_BIN ?? "tht"; if (typeof executable !== "string" || executable.length === 0 || /[\u0000-\u001f\u007f]/.test(executable)) throw invalid(); if (executable === "tht" || (pathStyle === "windows" && executable === "tht.exe")) return executable; const paths = pathStyle === "windows" ? win32 : posix; diff --git a/backend/src/server.ts b/backend/src/server.ts index df9f3536..06bf5f7f 100644 --- a/backend/src/server.ts +++ b/backend/src/server.ts @@ -1,5 +1,6 @@ import { buildApp, type AppWithAuthSessionStore } from "./app.js"; import { loadConfig } from "./config.js"; +import { formatStartupFailure } from "./startup-error.js"; const config = loadConfig(process.env); const app = buildApp(config) as AppWithAuthSessionStore; @@ -17,7 +18,7 @@ async function start(): Promise { console.log(`backend listening on ${address}`); } -void start().catch(() => { - console.error("backend startup failed"); +void start().catch((error: unknown) => { + console.error(formatStartupFailure(error)); process.exitCode = 1; }); diff --git a/backend/src/startup-error.ts b/backend/src/startup-error.ts new file mode 100644 index 00000000..d04ea6ce --- /dev/null +++ b/backend/src/startup-error.ts @@ -0,0 +1,12 @@ +const STARTUP_CAUSES = new Set([ + "auth_config_invalid", + "auth_session_store_invalid", + "workspace_registry_invalid", +]); + +/** Return one bounded machine cause; never include the original error text or stack. */ +export function formatStartupFailure(error: unknown): string { + const message = error instanceof Error ? error.message : ""; + const cause = STARTUP_CAUSES.has(message) ? message : "startup_unknown"; + return `backend startup failed: ${cause}`; +} diff --git a/backend/test/auth.test.ts b/backend/test/auth.test.ts index 20044f7d..dcbd1ac1 100644 --- a/backend/test/auth.test.ts +++ b/backend/test/auth.test.ts @@ -184,6 +184,43 @@ test("the session boundary exposes only exact health and authentication protocol expect((await app.inject({ method: "GET", url: "/auth/configured" })).statusCode).toBe(401); }); +test("the session boundary retains the exact loopback tht maintenance identity in configured auth modes", async () => { + const app = Fastify(); + app.addHook("preHandler", authenticateSession({ mode: "local" })); + app.get("/private", async (request) => getPrincipal(request)); + app.post("/private", async (request) => getPrincipal(request)); + const headers = { + "x-thoth-principal-issuer": "tht", + "x-thoth-principal-subject": "tht-maintenance", + "x-thoth-principal-display-name": "Tht maintenance", + "x-thoth-is-admin": "1", + }; + + for (const method of ["GET", "POST"] as const) { + const response = await app.inject({ method, url: "/private", headers, remoteAddress: "127.0.0.1" }); + expect(response.statusCode).toBe(200); + expect(response.json()).toMatchObject({ issuer: "tht", subject: "tht-maintenance", isAdmin: true }); + } +}); + +test("the session boundary rejects tht maintenance headers outside exact loopback provenance", async () => { + const app = Fastify(); + app.addHook("preHandler", authenticateSession({ mode: "local" })); + app.get("/private", async (request) => getPrincipal(request)); + const exact = { + "x-thoth-principal-issuer": "tht", + "x-thoth-principal-subject": "tht-maintenance", + "x-thoth-principal-display-name": "Tht maintenance", + "x-thoth-is-admin": "1", + }; + + expect((await app.inject({ method: "GET", url: "/private", headers: exact, remoteAddress: "172.30.0.9" })).statusCode).toBe(503); + expect((await app.inject({ + method: "GET", url: "/private", remoteAddress: "127.0.0.1", + headers: { ...exact, "x-thoth-principal-subject": "not-maintenance" }, + })).statusCode).toBe(503); +}); + test("the session boundary touches a valid cookie session through the bounded Task 7 store operation", async () => { const sessions = { resolve: vi.fn(async () => ({ diff --git a/backend/test/startup-error.test.ts b/backend/test/startup-error.test.ts new file mode 100644 index 00000000..bf1685bd --- /dev/null +++ b/backend/test/startup-error.test.ts @@ -0,0 +1,28 @@ +import { describe, expect, it } from "vitest"; +import { formatStartupFailure } from "../src/startup-error.js"; + +describe("formatStartupFailure", () => { + it.each([ + [new Error("auth_session_store_invalid"), "backend startup failed: auth_session_store_invalid"], + [new Error("auth_config_invalid"), "backend startup failed: auth_config_invalid"], + [new Error("workspace_registry_invalid"), "backend startup failed: workspace_registry_invalid"], + ])("emits only an allowlisted startup cause", (error, expected) => { + expect(formatStartupFailure(error)).toBe(expected); + }); + + it("collapses unknown errors without exposing their message, stack, token, or path", () => { + const error = new Error( + "EACCES password=plain-secret token=token-secret at /run/secrets/private-token", + ); + error.stack = "Error: raw failure\n at /app/backend/dist/server.js:42:1"; + + const formatted = formatStartupFailure(error); + + expect(formatted).toBe("backend startup failed: startup_unknown"); + for (const leaked of [ + "EACCES", "plain-secret", "token-secret", "/run/secrets", "raw failure", "server.js", + ]) { + expect(formatted).not.toContain(leaked); + } + }); +}); diff --git a/backend/test/windows-auth-storage.test.ts b/backend/test/windows-auth-storage.test.ts index fcd4546e..ac8fed9a 100644 --- a/backend/test/windows-auth-storage.test.ts +++ b/backend/test/windows-auth-storage.test.ts @@ -93,6 +93,29 @@ function bridgeForChild(child: FakeBridgeChild, pathStyle: "windows" | "posix" = } describe("Windows auth-storage bridge", () => { + test("uses a dedicated native storage executable without replacing the harness tht", async () => { + vi.stubEnv("THT_BIN", "/opt/venv/bin/tht"); + vi.stubEnv("THT_AUTH_STORAGE_BIN", "/usr/local/bin/tht-auth-storage"); + const calls: Array<{ executable: string }> = []; + const bridge = createPosixAuthStorageBridge({ + invoke: async (call) => { + calls.push(call); + return { + code: 0, + stdout: Buffer.from('{"version":1,"ok":true,"prepared":true}\n'), + stderr: Buffer.alloc(0), + }; + }, + }); + + await bridge.ensureLayout("/data/auth"); + + expect(calls).toHaveLength(1); + expect(calls[0]!.executable).toBe("/usr/local/bin/tht-auth-storage"); + expect(process.env.THT_BIN).toBe("/opt/venv/bin/tht"); + vi.unstubAllEnvs(); + }); + test("uses the same bounded hidden bridge to ensure a POSIX session layout", async () => { const calls: Array<{ executable: string; args: readonly string[]; input: Buffer; timeoutMs: number }> = []; const bridge = createPosixAuthStorageBridge({ diff --git a/docker/core-entrypoint.sh b/docker/core-entrypoint.sh index 6f9049cc..df2280e1 100755 --- a/docker/core-entrypoint.sh +++ b/docker/core-entrypoint.sh @@ -13,6 +13,10 @@ node /app/docker/ensure-pi-trust.mjs "${THT_HARNESS_DIR:-/app/harness}" cmd="${1:-server}" case "$cmd" in server) + [[ "${THT_AUTH_STATE_ROOT:-/data/auth}" == /data/auth ]] \ + || { printf '%s\n' 'authentication state root is invalid' >&2; exit 1; } + printf '%s\n' '{"version":1,"operation":"ensure-layout","root":"/data/auth"}' \ + | "${THT_AUTH_STORAGE_BIN:-/usr/local/bin/tht-auth-storage}" _auth-storage >/dev/null exec node /app/backend/dist/server.js ;; check) diff --git a/docker/core.Dockerfile b/docker/core.Dockerfile index 0d37203c..410eeff4 100644 --- a/docker/core.Dockerfile +++ b/docker/core.Dockerfile @@ -7,6 +7,14 @@ ARG IMAGE_VERSION=local # ---- Pinned Node source for the runtime binary and npm ---- FROM node:24.16.0-bookworm@sha256:40ad9f3064e67d6860b4bc3fe1880b2953934fd6320ada990e45fe0efa6badd7 AS node-runtime +# ---- Pinned native storage helper used by the authenticated backend ---- +FROM golang:1.26.5-bookworm@sha256:1ecb7edf62a0408027bd5729dfd6b1b8766e578e8df93995b225dfd0944eb651 AS tht-auth-storage-build +WORKDIR /src/tools/tht +COPY tools/tht/go.mod tools/tht/go.sum ./ +RUN go mod download +COPY tools/tht ./ +RUN CGO_ENABLED=0 go build -trimpath -ldflags='-s -w' -o /out/tht-auth-storage ./cmd/tht + # ---- Stage 0: locked Pi runtime ---- FROM node:24.16.0-bookworm@sha256:40ad9f3064e67d6860b4bc3fe1880b2953934fd6320ada990e45fe0efa6badd7 AS pi-runtime-build ARG PI_VERSION @@ -61,7 +69,9 @@ RUN useradd --create-home --uid 10001 --shell /bin/bash thoth # Docker copies these owned directories into newly-created named volumes, allowing the non-root # runtime user to create application settings, sessions, registry snapshots, state, and locks. RUN mkdir -p /home/thoth/.pi/agent /data/settings /data/sessions /data/workspace-registry /data/workspace-secrets \ - && chown -R thoth:thoth /home/thoth/.pi /data + /data/auth/sessions /data/auth/oidc \ + && chown -R thoth:thoth /home/thoth/.pi /data \ + && chmod 0700 /data/auth /data/auth/sessions /data/auth/oidc COPY harness/ /app/harness/ # Pi scrive lock/settings in .pi: ownership thoth per sopravvivere al rebuild @@ -91,14 +101,17 @@ COPY backend/package*.json /app/backend/ # Runtime Pi is installed only from the committed lockfile. The image exposes its immutable # executable directly, so no host Pi installation or writable global npm directory is needed. COPY --from=pi-runtime-build /opt/pi-runtime/node_modules /opt/pi-runtime/node_modules +COPY --from=tht-auth-storage-build /out/tht-auth-storage /usr/local/bin/tht-auth-storage RUN ln -s /opt/pi-runtime/node_modules/.bin/pi /usr/local/bin/pi \ - && test "$(pi --version)" = "$PI_VERSION" + && test "$(pi --version)" = "$PI_VERSION" \ + && test -x /usr/local/bin/tht-auth-storage ENV PATH="/opt/venv/bin:/usr/local/bin:$PATH" \ PI_VERSION="${PI_VERSION}" \ HOST=0.0.0.0 PORT=8787 \ THT_HARNESS_DIR=/app/harness \ THT_BIN=/opt/venv/bin/tht \ + THT_AUTH_STORAGE_BIN=/usr/local/bin/tht-auth-storage \ PI_BIN=pi \ HOME=/home/thoth diff --git a/scripts/test-default-compose.sh b/scripts/test-default-compose.sh index 4b968dfd..05207a90 100755 --- a/scripts/test-default-compose.sh +++ b/scripts/test-default-compose.sh @@ -24,8 +24,19 @@ if (services.join(",") !== "core,embedding,embedding-model-init,frontend,qdrant" if (/omics_portal|chirone|localllm_default|\/home\/chirone/i.test(JSON.stringify(config))) { throw new Error("default Compose contains application-specific coupling"); } -for (const volume of ["settings", "pi-state", "workspace-registry", "sessions", "auth-state", "qdrant-data", "embedding-models"]) { - if (!config.volumes || !config.volumes[volume]) throw new Error(`missing required volume: ${volume}`); +const expectedVolumes = [ + "auth-state", + "embedding-models", + "pi-state", + "qdrant-data", + "sessions", + "settings", + "workspace-registry", + "workspace-secrets", +]; +const actualVolumes = Object.keys(config.volumes || {}); +if (actualVolumes.join(",") !== expectedVolumes.join(",")) { + throw new Error(`unexpected ordered volume set: ${actualVolumes.join(",")}`); } const core = config.services.core; const frontend = config.services.frontend; diff --git a/scripts/unified-deployment-smoke.sh b/scripts/unified-deployment-smoke.sh index a8a6d669..636d40b2 100755 --- a/scripts/unified-deployment-smoke.sh +++ b/scripts/unified-deployment-smoke.sh @@ -166,6 +166,44 @@ task13_compose_logged() { task13_run_logged "$label" "${TASK13_COMPOSE[@]}" "$@" } +task13_report_core_startup_failure() { + local container_id state exit_code logs cause="startup failure is unclassified" + container_id="$(task13_compose ps --all -q core 2>/dev/null | head -n 1 || true)" + state="" + if [[ -n "$container_id" ]]; then + state="$(docker inspect --format '{{.State.Status}}:{{.State.ExitCode}}' "$container_id" 2>/dev/null || true)" + fi + exit_code="${state##*:}" + [[ "$exit_code" =~ ^[0-9]{1,3}$ ]] || exit_code="unknown" + logs="$(task13_compose logs --no-color --tail 100 core 2>/dev/null || true)" + case "$logs" in + *auth_session_store_invalid*|*auth*storage*request*failed*|*EACCES*auth*|*permission*auth*) + cause="authentication state storage is unavailable" + ;; + *auth_config_invalid*|*authentication*configuration*) + cause="authentication configuration is invalid" + ;; + *workspace_registry_invalid*|*workspace*registry*) + cause="workspace registry startup validation failed" + ;; + esac + printf 'Core startup cause: %s (exit code %s).\n' "$cause" "$exit_code" >&2 +} + +task13_compose_start_logged() { + local label="$1" + shift + task13_compose_files + if task13_bounded "$TASK13_COMMAND_TIMEOUT" "$label" \ + "${TASK13_COMPOSE[@]}" "$@" >>"$TASK13_LOG" 2>&1; then + return 0 + fi + TASK13_FAILURE_LOGGED=1 + printf 'Task 13 command failed: %s\n' "$label" >&2 + task13_report_core_startup_failure + return 1 +} + task13_write_environment() { local remote="$1" { @@ -296,6 +334,7 @@ services: - $TASK13_PI_MODELS:/home/thoth/.pi/agent/models.json:ro - $TASK13_PI_SETTINGS:/home/thoth/.pi/agent/settings.json:ro - workspace-registry:/data/workspace-registry + - workspace-secrets:/data/workspace-secrets - sessions:/data/sessions - $TASK13_AUTH_ROOT:/run/thothii-auth:ro - auth-state:/data/auth @@ -333,6 +372,9 @@ volumes: workspace-registry: labels: io.thothii.task13.run: "$TASK13_RUN_ID" + workspace-secrets: + labels: + io.thothii.task13.run: "$TASK13_RUN_ID" sessions: labels: io.thothii.task13.run: "$TASK13_RUN_ID" @@ -362,6 +404,9 @@ EOF task13_write_server_fixture_files() { local data_root pi_root registry_root remote_path workspace_path + TASK13_OIDC_SERVER="${TASK13_OIDC_SERVER:-$TASK13_TMP/fake-oidc.mjs}" + TASK13_OIDC_CERT="${TASK13_OIDC_CERT:-$TASK13_TMP/fake-oidc-cert.pem}" + TASK13_OIDC_KEY="${TASK13_OIDC_KEY:-$TASK13_TMP/fake-oidc-key.pem}" printf '{}\n' >"$TASK13_PI_AUTH" printf 'THT_MODEL_API_KEY=%s\nTHT_OIDC_CLIENT_SECRET=%s\nTHT_AUTHENTIK_API_TOKEN=%s\n' \ "$TASK13_SECRET_VALUE" "$TASK13_OIDC_CLIENT_SECRET" "$TASK13_AUTHENTIK_API_TOKEN" >"$TASK13_SECRETS" @@ -376,6 +421,63 @@ EOF chmod 0600 "$TASK13_SECRETS" "$TASK13_SESSION_RUNTIME_PASSWORD" \ "$TASK13_SESSION_MIGRATOR_PASSWORD_FILE" + task13_run_logged "create scoped fake OIDC certificate" openssl req -x509 -newkey rsa:2048 \ + -sha256 -nodes -days 1 -subj '/CN=task13-fake-oidc' \ + -addext 'subjectAltName=DNS:task13-fake-oidc' \ + -keyout "$TASK13_OIDC_KEY" -out "$TASK13_OIDC_CERT" + chmod 0600 "$TASK13_OIDC_KEY" + chmod 0644 "$TASK13_OIDC_CERT" + cat >"$TASK13_OIDC_SERVER" <<'EOF' +import { createPublicKey, generateKeyPairSync } from "node:crypto"; +import { readFileSync } from "node:fs"; +import https from "node:https"; + +const origin = "https://task13-fake-oidc:9443"; +const issuer = `${origin}/application/o/task13/`; +const expectedToken = process.env.TASK13_AUTHENTIK_API_TOKEN; +const { publicKey } = generateKeyPairSync("rsa", { modulusLength: 2048 }); +const jwk = { ...createPublicKey(publicKey).export({ format: "jwk" }), kid: "task13", use: "sig", alg: "RS256" }; +const send = (response, status, body) => { + const payload = JSON.stringify(body); + response.writeHead(status, { "content-type": "application/json", "content-length": Buffer.byteLength(payload) }); + response.end(payload); +}; + +const server = https.createServer({ + cert: readFileSync("/fixtures/oidc-cert.pem"), + key: readFileSync("/fixtures/oidc-key.pem"), +}, (request, response) => { + const target = new URL(request.url ?? "/", origin); + if (target.pathname === "/health") return send(response, 200, { status: "ok" }); + if (target.pathname.includes(".well-known/openid-configuration")) { + return send(response, 200, { + issuer, + authorization_endpoint: `${origin}/authorize`, + token_endpoint: `${origin}/token`, + jwks_uri: `${origin}/jwks`, + response_types_supported: ["code"], + subject_types_supported: ["public"], + id_token_signing_alg_values_supported: ["RS256"], + }); + } + if (target.pathname === "/jwks") return send(response, 200, { keys: [jwk] }); + if (target.pathname === "/api/v3/core/groups/") { + if (request.headers.authorization !== `Bearer ${expectedToken}`) return send(response, 401, { detail: "unauthorized" }); + const name = target.searchParams.get("name") ?? ""; + console.log(`group:${name}`); + const configured = name === "task13-users" || name === "task13-admins"; + return send(response, 200, { + pagination: { next: null }, + results: configured ? [{ name }, { name: "task13-unrelated" }] : [{ name: "task13-unrelated" }], + }); + } + return send(response, 404, { error: "not_found" }); +}); + +server.listen(9443, "0.0.0.0"); +EOF + chmod 0644 "$TASK13_OIDC_SERVER" + cat >"$TASK13_SERVER_WORKSPACE_CONFIG" <<'EOF' language: en session_storage: @@ -417,6 +519,7 @@ services: labels: io.thothii.task13.run: "$TASK13_RUN_ID" environment: + NODE_EXTRA_CA_CERTS: /fixtures/task13-oidc-ca.pem THT_WS_TASK13_SMOKE_DWH_TRANSPORT: postgres_direct THT_WS_TASK13_SMOKE_DWH_HOST: dwh.task13.invalid THT_WS_TASK13_SMOKE_DWH_PORT: "5432" @@ -427,6 +530,7 @@ services: volumes: - $remote_path:/fixtures/remote.git:ro - $TASK13_SESSION_RUNTIME_PASSWORD:/run/secrets/task13-runtime-password:ro + - $TASK13_OIDC_CERT:/fixtures/task13-oidc-ca.pem:ro frontend: image: $TASK13_FRONTEND_IMAGE build: @@ -639,15 +743,15 @@ task13_configure_local_authentication() { task13_configure_server_oidc_authentication() { task13_run_logged "configure fake server OIDC authentication" "$TASK13_THT" --installation "$TASK13_INSTALLATION" auth configure \ --mode oidc --public-url "https://task13.example.invalid" \ - --issuer "https://task13-fake-oidc.invalid/application/o/task13/" --client-id task13-smoke-client \ - --authentik-base-url "https://task13-fake-authentik.invalid" --user-group task13-users --admin-group task13-admins + --issuer "https://task13-fake-oidc:9443/application/o/task13/" --client-id task13-smoke-client \ + --authentik-base-url "https://task13-fake-oidc:9443" --user-group task13-users --admin-group task13-admins } task13_start_stack() { printf '== Build and start isolated local Compose distribution ==\n' task13_assert_rendered_contract task13_compose_logged "build local Compose images" build --pull - task13_compose_logged "start local Compose distribution" up --detach --wait --wait-timeout 120 + task13_compose_start_logged "start local Compose distribution" up --detach --wait --wait-timeout 120 TASK13_NETWORK="$(docker network ls \ --filter "label=com.docker.compose.project=$TASK13_PROJECT" \ --filter 'label=com.docker.compose.network=thothii' --format '{{.Name}}')" @@ -674,7 +778,36 @@ task13_start_server_stack() { printf '== Build and start isolated Linux server profile ==\n' task13_assert_rendered_contract task13_compose_logged "build server Compose images" build --pull core frontend - task13_compose_logged "start server Compose distribution" \ + task13_compose_logged "create server Compose resources" create core frontend + TASK13_NETWORK="$(docker network ls \ + --filter "label=com.docker.compose.project=$TASK13_PROJECT" \ + --filter 'label=com.docker.compose.network=thothii' --format '{{.Name}}')" + [[ -n "$TASK13_NETWORK" && "$TASK13_NETWORK" != *$'\n'* ]] \ + || task13_fail "isolated server Compose network was not resolved" + task13_run_logged "start scoped fake OIDC provider" docker run --detach \ + --name "$TASK13_OIDC_CONTAINER" \ + --label "io.thothii.task13.run=$TASK13_RUN_ID" \ + --network "$TASK13_NETWORK" --network-alias task13-fake-oidc \ + --user "$(id -u):$(id -g)" \ + --env "TASK13_AUTHENTIK_API_TOKEN=$TASK13_AUTHENTIK_API_TOKEN" \ + --env NODE_EXTRA_CA_CERTS=/fixtures/oidc-cert.pem \ + --entrypoint node \ + --volume "$TASK13_OIDC_SERVER:/fixtures/fake-oidc.mjs:ro" \ + --volume "$TASK13_OIDC_CERT:/fixtures/oidc-cert.pem:ro" \ + --volume "$TASK13_OIDC_KEY:/fixtures/oidc-key.pem:ro" \ + "$TASK13_CORE_IMAGE" /fixtures/fake-oidc.mjs + for _attempt in $(seq 1 30); do + if docker exec "$TASK13_OIDC_CONTAINER" node -e \ + "fetch('https://task13-fake-oidc:9443/health',{signal:AbortSignal.timeout(3000)}).then(r=>{if(!r.ok)process.exit(1)}).catch(()=>process.exit(1))" \ + >>"$TASK13_LOG" 2>&1; then + break + fi + sleep 1 + done + docker exec "$TASK13_OIDC_CONTAINER" node -e \ + "fetch('https://task13-fake-oidc:9443/health',{signal:AbortSignal.timeout(3000)}).then(r=>{if(!r.ok)process.exit(1)}).catch(()=>process.exit(1))" \ + >>"$TASK13_LOG" 2>&1 || task13_log_failure "scoped fake OIDC provider readiness" + task13_compose_start_logged "start server Compose distribution" \ up --detach --wait --wait-timeout 120 core frontend } @@ -729,6 +862,116 @@ task13_assert_local_auth_lifecycle() { [[ "$unauthenticated" == 401 ]] || task13_fail "local logout did not revoke the remembered session" } +task13_create_admin_session() { + local frontend login_body me + frontend="$(task13_frontend_address)" + TASK13_ADMIN_COOKIE="$TASK13_TMP/operations-admin.cookies" + login_body="$TASK13_TMP/operations-admin-login.json" + printf '{"username":"%s","password":"%s","remember":true}' \ + "$TASK13_AUTH_ADMIN" "$TASK13_AUTH_PASSWORD" >"$login_body" + chmod 0600 "$login_body" + task13_run_logged "create authenticated smoke administration session" curl \ + --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" --max-time "$TASK13_CURL_MAX_TIME" \ + --fail --silent --show-error --cookie-jar "$TASK13_ADMIN_COOKIE" \ + -H "Origin: http://$frontend" -H 'content-type: application/json' --data-binary "@$login_body" \ + "http://$frontend/api/auth/local/login" + me="$(curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" --max-time "$TASK13_CURL_MAX_TIME" \ + --fail --silent --show-error --cookie "$TASK13_ADMIN_COOKIE" "http://$frontend/api/me")" + TASK13_ADMIN_CSRF="$(node -e 'const value=JSON.parse(process.argv[1]); if(typeof value.csrfToken!=="string") process.exit(1); process.stdout.write(value.csrfToken)' "$me")" \ + || task13_fail "authenticated smoke administration session lacks CSRF state" +} + +task13_authenticated_get() { + local path="$1" frontend + frontend="$(task13_frontend_address)" + curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" --max-time "$TASK13_CURL_MAX_TIME" \ + --fail --silent --show-error --cookie "$TASK13_ADMIN_COOKIE" "http://$frontend/api/$path" +} + +task13_authenticated_post() { + local path="$1" frontend + frontend="$(task13_frontend_address)" + curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" --max-time 15 \ + --fail --silent --show-error --cookie "$TASK13_ADMIN_COOKIE" \ + -H "Origin: http://$frontend" -H "x-thothii-csrf: $TASK13_ADMIN_CSRF" \ + -X POST "http://$frontend/api/$path" +} + +task13_assert_local_restore_reauthentication() { + local frontend archive login_body cookie_before cookie_after me status_before status_after + frontend="$(task13_frontend_address)" + archive="$TASK13_TMP/local-restore-source.zip" + login_body="$TASK13_TMP/local-restore-login.json" + cookie_before="$TASK13_TMP/local-restore-before.cookies" + cookie_after="$TASK13_TMP/local-restore-after.cookies" + printf '{"username":"%s","password":"%s","remember":true}' \ + "$TASK13_AUTH_ADMIN" "$TASK13_AUTH_PASSWORD" >"$login_body" + chmod 0600 "$login_body" + + task13_run_logged "create pre-backup browser session" curl \ + --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" --max-time "$TASK13_CURL_MAX_TIME" \ + --fail --silent --show-error --cookie-jar "$cookie_before" \ + -H "Origin: http://$frontend" -H 'content-type: application/json' --data-binary "@$login_body" \ + "http://$frontend/api/auth/local/login" + task13_compose_logged "stop local stack for backup" stop + task13_run_logged "create real default-custody backup" "$TASK13_THT" \ + --installation "$TASK13_INSTALLATION" backup --output "$archive" + python3 - "$archive" <<'PY' +import json +import sys +import zipfile + +with zipfile.ZipFile(sys.argv[1]) as archive: + manifest = json.loads(archive.read("manifest.json")) +volumes = [item["logical_name"] for item in manifest["volumes"]] +if volumes != ["embedding-models", "pi-state", "qdrant-data", "sessions", "settings", "workspace-registry", "workspace-secrets"]: + raise SystemExit(f"unexpected backup volume custody: {volumes}") +entries = manifest["entries"] +if any(item.get("logical_name") == "auth-state" or "/data/auth" in item.get("source_path", "") for item in entries): + raise SystemExit("default backup contains authentication runtime state") +auth = [item for item in entries if item["path"].startswith("authentication-secrets/")] +if len(auth) != 1 or not auth[0]["path"].endswith("-auth.yaml") or auth[0]["archived"]: + raise SystemExit("default backup auth custody is not an auth.yaml reference only") +if any(item["path"].endswith("users.yaml") for item in entries): + raise SystemExit("default backup contains users.yaml") +PY + + task13_compose_start_logged "restart local stack before restore" up --detach --wait --wait-timeout 120 + status_before="$(curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" --max-time "$TASK13_CURL_MAX_TIME" \ + --silent --output /dev/null --write-out '%{http_code}' --cookie "$cookie_before" "http://$frontend/api/me")" + [[ "$status_before" == 200 ]] || task13_fail "pre-backup browser session did not survive an ordinary stop/start" + task13_run_logged "create post-backup browser session" curl \ + --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" --max-time "$TASK13_CURL_MAX_TIME" \ + --fail --silent --show-error --cookie-jar "$cookie_after" \ + -H "Origin: http://$frontend" -H 'content-type: application/json' --data-binary "@$login_body" \ + "http://$frontend/api/auth/local/login" + me="$(curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" --max-time "$TASK13_CURL_MAX_TIME" \ + --fail --silent --show-error --cookie "$cookie_after" "http://$frontend/api/me")" + node -e 'const value=JSON.parse(process.argv[1]); if(value.issuer!=="local"||value.session?.remembered!==true) process.exit(1)' "$me" \ + || task13_fail "post-backup browser session was not authenticated" + task13_compose_logged "seed pending OIDC state excluded from restore" exec -T core sh -ceu \ + 'printf %s "{}" > /data/auth/oidc/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.json && chmod 0600 /data/auth/oidc/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.json && test "$(find /data/auth/sessions -type f | wc -l | tr -d " ")" -ge 2' + + task13_compose_logged "stop local stack for restore" stop + task13_run_logged "perform real production restore" "$TASK13_THT" \ + --installation "$TASK13_INSTALLATION" restore "$archive" --yes + task13_compose_start_logged "start restored local stack" up --detach --wait --wait-timeout 120 + status_before="$(curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" --max-time "$TASK13_CURL_MAX_TIME" \ + --silent --output /dev/null --write-out '%{http_code}' --cookie "$cookie_before" "http://$frontend/api/me")" + status_after="$(curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" --max-time "$TASK13_CURL_MAX_TIME" \ + --silent --output /dev/null --write-out '%{http_code}' --cookie "$cookie_after" "http://$frontend/api/me")" + [[ "$status_before" == 401 && "$status_after" == 401 ]] \ + || task13_fail "restore did not force every independent browser session to reauthenticate" + task13_compose_logged "verify private empty restored auth state" exec -T core sh -ceu ' + test "$(stat -c %a /data/auth)" = 700 + test "$(stat -c %a /data/auth/sessions)" = 700 + test "$(stat -c %a /data/auth/oidc)" = 700 + test "$(stat -c %u /data/auth)" = "$(id -u)" + test -z "$(find /data/auth/sessions /data/auth/oidc -mindepth 1 -print -quit)" + ' + task13_create_admin_session +} + task13_assert_runtime() { local frontend expected_pi actual_pi core_id frontend="$(task13_frontend_address)" @@ -745,8 +988,10 @@ task13_assert_runtime() { 'command -v pi >/dev/null' task13_compose_logged "core Docker socket isolation" exec -T core sh -ceu \ 'test ! -e /var/run/docker.sock' - task13_compose_logged "workspace registry bootstrap" exec -T core \ - curl --connect-timeout 3 --max-time 10 -fsS http://127.0.0.1:8787/workspace-registry/status + task13_assert_local_auth_lifecycle + task13_create_admin_session + task13_authenticated_get workspace-registry/status >/dev/null \ + || task13_fail "authenticated workspace registry bootstrap failed" task13_compose_logged "active workspace registry state" exec -T core sh -ceu \ 'test -f /data/workspace-registry/state/active.json' task13_compose_logged "mounted Pi auth readability" exec -T core sh -ceu \ @@ -757,7 +1002,6 @@ task13_assert_runtime() { [[ "$(docker inspect --format '{{ index .Config.Labels "io.thothii.task13.run" }}' "$core_id")" == "$TASK13_RUN_ID" ]] \ || task13_fail "core lacks the explicit Task 13 resource label" task13_assert_maintenance_auth_isolation - task13_assert_local_auth_lifecycle task13_run_logged "tht Pi doctor" "$TASK13_THT" --installation "$TASK13_INSTALLATION" pi doctor } @@ -799,7 +1043,7 @@ task13_report_server_workspace_failure() { } task13_assert_server_runtime() { - local frontend unauthenticated trusted_header_status session_status diagnostics diagnostic_status core_id frontend_id + local frontend unauthenticated trusted_header_status session_status diagnostics status provider_requests core_id frontend_id local expected_core_image expected_frontend_image frontend="$(task13_frontend_address)" task13_run_logged "server frontend health" curl \ @@ -857,22 +1101,32 @@ task13_assert_server_runtime() { if grep -Fq "$TASK13_SECRET_VALUE" "$TASK13_TMP/server-sessions.out"; then task13_fail "server session failure exposed the fixture secret" fi + status="$TASK13_TMP/server-auth-status.json" + task13_run_logged "server static OIDC status" "$TASK13_THT" --installation "$TASK13_INSTALLATION" auth status --json + "$TASK13_THT" --installation "$TASK13_INSTALLATION" auth status --json >"$status" + node -e 'const value=JSON.parse(require("fs").readFileSync(process.argv[1], "utf8")); if(value.mode!=="oidc"||typeof value.configRevision!=="string"||value.configRevision.length!==64) process.exit(1)' "$status" \ + || task13_fail "server static OIDC status was not valid" diagnostics="$TASK13_TMP/server-auth-diagnostics.json" - set +e - "$TASK13_THT" --installation "$TASK13_INSTALLATION" auth check --json >"$diagnostics" 2>>"$TASK13_LOG" - diagnostic_status=$? - set -e - [[ "$diagnostic_status" == 1 ]] || task13_fail "fake OIDC diagnostics did not fail closed" - node -e 'const value=JSON.parse(require("fs").readFileSync(process.argv[1], "utf8")); if(value.mode!=="oidc"||value.ready!==false||!value.checks.some((item)=>item.code==="oidc_discovery_unreachable")) process.exit(1)' "$diagnostics" \ - || task13_fail "fake OIDC fixture did not produce the expected static diagnostic" + task13_run_logged "server live OIDC diagnostics" "$TASK13_THT" --installation "$TASK13_INSTALLATION" auth check --json + "$TASK13_THT" --installation "$TASK13_INSTALLATION" auth check --json >"$diagnostics" + node -e 'const value=JSON.parse(require("fs").readFileSync(process.argv[1], "utf8")); if(value.mode!=="oidc"||value.ready!==true||value.checks.length!==1||value.checks[0].code!=="auth_ready") process.exit(1)' "$diagnostics" \ + || task13_fail "scoped fake OIDC provider did not pass live production diagnostics" + provider_requests="$(docker logs "$TASK13_OIDC_CONTAINER" 2>>"$TASK13_LOG")" + [[ "$(grep -Fc 'group:task13-users' <<<"$provider_requests")" -ge 1 ]] \ + || task13_fail "live OIDC diagnostics did not verify the mandatory user group" + [[ "$(grep -Fc 'group:task13-admins' <<<"$provider_requests")" -ge 1 ]] \ + || task13_fail "live OIDC diagnostics did not verify the mandatory administrator group" + if grep -Fq 'group:task13-unrelated' <<<"$provider_requests" \ + || grep -Fq 'task13-unrelated' "$diagnostics"; then + task13_fail "live OIDC diagnostics queried or warned about an unrelated group" + fi if grep -Fq "$TASK13_OIDC_CLIENT_SECRET" "$diagnostics" || grep -Fq "$TASK13_AUTHENTIK_API_TOKEN" "$diagnostics"; then task13_fail "OIDC diagnostics exposed a fixture secret" fi } task13_registry_status() { - task13_compose exec -T core curl --connect-timeout 3 --max-time 10 -fsS \ - http://127.0.0.1:8787/workspace-registry/status + task13_authenticated_get workspace-registry/status } task13_registry_head() { @@ -914,8 +1168,7 @@ task13_prepare_persistence() { TASK13_INITIAL_MOUNTS="$(task13_mount_fingerprint)" TASK13_INITIAL_HEAD="$(task13_active_registry_head)" [[ "$TASK13_INITIAL_HEAD" =~ ^[0-9a-f]{40}$ ]] || task13_fail "initial registry head is invalid" - task13_compose exec -T core curl --connect-timeout 3 --max-time 10 -fsS \ - http://127.0.0.1:8787/workspaces \ + task13_authenticated_get workspaces \ | grep -Fq 'Task 13 Smoke' || task13_fail "initial workspace is unavailable" } @@ -939,8 +1192,8 @@ task13_registry_lifecycle() { task13_commit_registry_change 'Update Task 13 workspace metadata' task13_write_environment /fixtures/remote.git task13_compose_logged "online Compose recreation" up --detach --force-recreate --wait --wait-timeout 120 - task13_compose exec -T core curl --connect-timeout 3 --max-time 15 -fsS -X POST http://127.0.0.1:8787/workspace-registry/pull >/dev/null - task13_compose exec -T core curl --connect-timeout 3 --max-time 10 -fsS http://127.0.0.1:8787/workspaces | grep -Fq 'Task 13 Smoke Updated' || task13_fail "valid Git update was not activated" + task13_authenticated_post workspace-registry/pull >/dev/null + task13_authenticated_get workspaces | grep -Fq 'Task 13 Smoke Updated' || task13_fail "valid Git update was not activated" valid_head="$(task13_active_registry_head)" [[ "$valid_head" =~ ^[0-9a-f]{40}$ && "$valid_head" != "$TASK13_INITIAL_HEAD" ]] || task13_fail "valid Git update did not advance the registry head" TASK13_INITIAL_HEAD="$valid_head" @@ -950,7 +1203,7 @@ task13_registry_lifecycle() { ' printf 'guide v2\n' >"$TASK13_SEED/$TASK13_WORKSPACE_ID/evidence/guide.md" task13_commit_registry_change 'Update Task 13 workspace evidence only' - task13_compose exec -T core curl --connect-timeout 3 --max-time 15 -fsS -X POST http://127.0.0.1:8787/workspace-registry/pull >/dev/null + task13_authenticated_post workspace-registry/pull >/dev/null evidence_head="$(task13_active_registry_head)" [[ "$evidence_head" =~ ^[0-9a-f]{40}$ && "$evidence_head" != "$valid_head" ]] || task13_fail "content-only Git Evidence update did not advance the registry head" TASK13_INITIAL_HEAD="$evidence_head" @@ -960,14 +1213,14 @@ task13_registry_lifecycle() { ' task13_replace_once "$TASK13_SEED/thoth-workspaces.yaml" 'name: Task 13 Smoke Updated' 'name: Task 13 Smoke Drift' task13_commit_registry_change 'Break Task 13 workspace metadata parity' - if task13_compose exec -T core curl --connect-timeout 3 --max-time 15 -fsS -X POST http://127.0.0.1:8787/workspace-registry/pull >>"$TASK13_LOG" 2>&1; then + if task13_authenticated_post workspace-registry/pull >>"$TASK13_LOG" 2>&1; then task13_fail "registry accepted catalog/descriptor metadata mismatch" fi [[ "$(task13_active_registry_head)" == "$TASK13_INITIAL_HEAD" ]] || task13_fail "metadata mismatch replaced the valid registry head" - task13_compose exec -T core curl --connect-timeout 3 --max-time 10 -fsS http://127.0.0.1:8787/workspaces | grep -Fq 'Task 13 Smoke Updated' || task13_fail "metadata mismatch displaced the valid workspace" + task13_authenticated_get workspaces | grep -Fq 'Task 13 Smoke Updated' || task13_fail "metadata mismatch displaced the valid workspace" task13_replace_once "$TASK13_SEED/thoth-workspaces.yaml" 'name: Task 13 Smoke Drift' 'name: Task 13 Smoke Updated' task13_commit_registry_change 'Restore Task 13 workspace metadata parity' - task13_compose exec -T core curl --connect-timeout 3 --max-time 15 -fsS -X POST http://127.0.0.1:8787/workspace-registry/pull >/dev/null + task13_authenticated_post workspace-registry/pull >/dev/null repaired_head="$(task13_active_registry_head)" [[ "$repaired_head" =~ ^[0-9a-f]{40}$ && "$repaired_head" != "$TASK13_INITIAL_HEAD" ]] || task13_fail "metadata repair did not restore a fresh valid registry head" TASK13_INITIAL_HEAD="$repaired_head" @@ -983,14 +1236,14 @@ task13_registry_lifecycle() { mkdir -p "$TASK13_SEED/orphan/evidence" printf 'orphan guide\n' >"$TASK13_SEED/orphan/evidence/guide.md" task13_commit_registry_change 'Add orphan Task 13 workspace directory' - if task13_compose exec -T core curl --connect-timeout 3 --max-time 15 -fsS -X POST http://127.0.0.1:8787/workspace-registry/pull >>"$TASK13_LOG" 2>&1; then + if task13_authenticated_post workspace-registry/pull >>"$TASK13_LOG" 2>&1; then task13_fail "registry accepted orphan Task 13 descriptor directory" fi [[ "$(task13_active_registry_head)" == "$TASK13_INITIAL_HEAD" ]] || task13_fail "orphan descriptor directory replaced the valid registry head" - task13_compose exec -T core curl --connect-timeout 3 --max-time 10 -fsS http://127.0.0.1:8787/workspaces | grep -Fq 'Task 13 Smoke Updated' || task13_fail "orphan descriptor displaced the valid workspace" + task13_authenticated_get workspaces | grep -Fq 'Task 13 Smoke Updated' || task13_fail "orphan descriptor displaced the valid workspace" rm -rf "$TASK13_SEED/orphan" task13_commit_registry_change 'Remove orphan Task 13 workspace directory' - task13_compose exec -T core curl --connect-timeout 3 --max-time 15 -fsS -X POST http://127.0.0.1:8787/workspace-registry/pull >/dev/null + task13_authenticated_post workspace-registry/pull >/dev/null TASK13_INITIAL_HEAD="$(task13_active_registry_head)" printf '== Reject the retired flat workspace layout and retain the valid snapshot == @@ -999,11 +1252,11 @@ task13_registry_lifecycle() { cp "$TASK13_ROOT/scripts/fixtures/workspace-registry-task13.yaml" "$TASK13_SEED/workspaces/$TASK13_WORKSPACE_ID.yaml" printf 'legacy guide\n' >"$TASK13_SEED/workspace-content/$TASK13_WORKSPACE_ID/evidence/guide.md" task13_commit_registry_change 'Reintroduce retired flat Task 13 workspace layout' - if task13_compose exec -T core curl --connect-timeout 3 --max-time 15 -fsS -X POST http://127.0.0.1:8787/workspace-registry/pull >>"$TASK13_LOG" 2>&1; then + if task13_authenticated_post workspace-registry/pull >>"$TASK13_LOG" 2>&1; then task13_fail "registry accepted the retired flat Task 13 workspace layout" fi [[ "$(task13_active_registry_head)" == "$TASK13_INITIAL_HEAD" ]] || task13_fail "retired flat workspace layout replaced the valid registry head" - task13_compose exec -T core curl --connect-timeout 3 --max-time 10 -fsS http://127.0.0.1:8787/workspaces | grep -Fq 'Task 13 Smoke Updated' || task13_fail "retired flat workspace layout displaced the valid workspace" + task13_authenticated_get workspaces | grep -Fq 'Task 13 Smoke Updated' || task13_fail "retired flat workspace layout displaced the valid workspace" task13_assert_sentinels } @@ -1059,8 +1312,7 @@ task13_update_rollback() { task13_assert_sentinels task13_run_logged "post-rollback tht doctor" \ "$TASK13_THT" --installation "$TASK13_INSTALLATION" pi doctor - task13_compose exec -T core curl --connect-timeout 3 --max-time 10 -fsS \ - http://127.0.0.1:8787/workspaces \ + task13_authenticated_get workspaces \ | grep -Fq 'Task 13 Smoke' || task13_fail "rollback lost the active workspace" } @@ -1174,6 +1426,7 @@ task13_cleanup() { fi task13_remove_labeled_container "${TASK13_BAD_CANDIDATE_CONTAINER:-}" || cleanup_rc=1 task13_remove_labeled_container "${TASK13_LLM_CONTAINER:-}" || cleanup_rc=1 + task13_remove_labeled_container "${TASK13_OIDC_CONTAINER:-}" || cleanup_rc=1 if [[ -n "${TASK13_PROJECT:-}" && -n "${TASK13_ROOT:-}" && -f "${TASK13_OVERRIDE:-}" ]]; then if task13_assert_project_ownership >>"${TASK13_LOG:-/dev/null}" 2>&1; then task13_compose_files @@ -1304,6 +1557,40 @@ task13_self_test_server_workspace_diagnostics() { || task13_fail "server diagnostics did not sanitize response and core logs" } +task13_self_test_core_startup_diagnostics() { + local output + TASK13_SECRET_VALUE="fixture-known-secret" + + task13_compose() { + case "$*" in + "ps --all -q core") printf '%s\n' 'task13-core-id' ;; + "logs --no-color --tail 100 core") + printf '%s\n' \ + 'Error: EACCES: permission denied, mkdir /data/auth/sessions' \ + 'password=plain-secret token=fixture-known-secret' \ + 'secret path: /run/secrets/private-token' \ + ' at createFileAuthSessionStore (/app/backend/dist/auth/session-store.js:101:9)' + ;; + *) task13_fail "startup diagnostics requested an unexpected Compose command: $*" ;; + esac + } + docker() { + [[ "$*" == "inspect --format {{.State.Status}}:{{.State.ExitCode}} task13-core-id" ]] \ + || task13_fail "startup diagnostics requested an unexpected Docker command: $*" + printf '%s\n' 'exited:1' + } + + output="$(task13_report_core_startup_failure 2>&1)" + unset -f task13_compose docker + + [[ "$output" == 'Core startup cause: authentication state storage is unavailable (exit code 1).' ]] \ + || task13_fail "startup diagnostics emitted a non-allowlisted cause: $output" + for leaked in EACCES permission /data/auth /run/secrets plain-secret fixture-known-secret \ + createFileAuthSessionStore session-store.js; do + [[ "$output" != *"$leaked"* ]] || task13_fail "startup diagnostics leaked $leaked" + done +} + task13_self_test_cleanup_ownership() { local calls foreign_error owned_name foreign_name calls="$(mktemp "${TMPDIR:-/tmp}/thothii-task13-cleanup-contract.XXXXXX")" @@ -1664,6 +1951,7 @@ task13_self_test_source_contract() { task13_self_test() { task13_self_test_sanitizer + task13_self_test_core_startup_diagnostics task13_self_test_server_workspace_diagnostics task13_self_test_cleanup_ownership task13_self_test_image_cleanup_ownership @@ -1695,6 +1983,7 @@ task13_self_test_case() { windows) task13_self_test_windows_release_contract ;; server) task13_self_test_server_release_contract ;; server-diagnostics) task13_self_test_server_workspace_diagnostics ;; + startup-diagnostics) task13_self_test_core_startup_diagnostics ;; *) task13_fail "unknown Task 13 self-test case: $1" ;; esac } @@ -1777,8 +2066,12 @@ task13_initialize() { TASK13_PI_MODELS="$TASK13_TMP/models.json" TASK13_PI_SETTINGS="$TASK13_TMP/pi-settings.json" TASK13_LLM_SERVER="$TASK13_TMP/fake-llm.mjs" + TASK13_OIDC_SERVER="$TASK13_TMP/fake-oidc.mjs" + TASK13_OIDC_CERT="$TASK13_TMP/fake-oidc-cert.pem" + TASK13_OIDC_KEY="$TASK13_TMP/fake-oidc-key.pem" TASK13_THT_DIR="$TASK13_TMP/tht" TASK13_LLM_CONTAINER="$TASK13_PROJECT-llm" + TASK13_OIDC_CONTAINER="$TASK13_PROJECT-oidc" TASK13_BAD_CANDIDATE_CONTAINER="$TASK13_PROJECT-bad-candidate" TASK13_CORE_IMAGE="task13-core-$TASK13_RUN_ID:local" TASK13_FRONTEND_IMAGE="task13-frontend-$TASK13_RUN_ID:local" @@ -1799,7 +2092,7 @@ task13_initialize() { } task13_require_tools() { - for command in bash git docker curl node sed awk grep rg sort; do + for command in bash git docker curl node openssl python3 sed awk grep rg sort; do command -v "$command" >/dev/null 2>&1 || task13_fail "$command is required" done task13_run_logged "Docker daemon readiness" docker info @@ -1823,6 +2116,7 @@ task13_smoke_main() { task13_assert_project_ownership task13_assert_built_image_ownership task13_assert_runtime + task13_assert_local_restore_reauthentication task13_prepare_persistence if [[ "$mode" == full ]]; then task13_registry_lifecycle diff --git a/tools/tht/internal/backup/create.go b/tools/tht/internal/backup/create.go index 080e6443..86acf36e 100644 --- a/tools/tht/internal/backup/create.go +++ b/tools/tht/internal/backup/create.go @@ -457,26 +457,6 @@ func inspectRequiredVolumes(ctx context.Context, runner archiveRunner, rendered } func imageIdentities(ctx context.Context, installation config.Installation, runner archiveRunner, rendered renderedCompose) ([]ImageIdentity, error) { - result, err := runner.Run(ctx, installation.ComposeArgs("images", "--format", "json"), nil) - if err != nil { - return nil, dockerError("inspect image identities", result, err) - } - ids := map[string]string{} - decoder := json.NewDecoder(strings.NewReader(result.Stdout)) - for { - var item struct { - Service string `json:"Service"` - ID string `json:"ID"` - } - err := decoder.Decode(&item) - if errors.Is(err, io.EOF) { - break - } - if err != nil || item.Service == "" { - return nil, errors.New("Docker Compose returned invalid image identities") - } - ids[item.Service] = item.ID - } services := make([]string, 0, len(rendered.Services)) for name, definition := range rendered.Services { if definition.Image != "" { @@ -486,11 +466,70 @@ func imageIdentities(ctx context.Context, installation config.Installation, runn sort.Strings(services) images := make([]ImageIdentity, 0, len(services)) for _, name := range services { - images = append(images, ImageIdentity{Service: name, Reference: rendered.Services[name].Image, ID: ids[name]}) + result, err := runner.Run(ctx, installation.ComposeArgs("images", "--format", "json", name), nil) + if err != nil { + return nil, dockerError("inspect image identities", result, err) + } + inspected, err := decodeComposeImageIdentities(result.Stdout) + if err != nil { + return nil, err + } + matching := make([]composeImageIdentity, 0, len(inspected)) + for _, item := range inspected { + if item.Service == "" || item.Service == name { + matching = append(matching, item) + } + } + if len(matching) > 1 { + return nil, errors.New("Docker Compose returned invalid image identities") + } + id := "" + if len(matching) == 1 { + id = matching[0].ID + } + images = append(images, ImageIdentity{Service: name, Reference: rendered.Services[name].Image, ID: id}) } return images, nil } +type composeImageIdentity struct { + Service string `json:"Service"` + ContainerName string `json:"ContainerName"` + ID string `json:"ID"` +} + +func decodeComposeImageIdentities(value string) ([]composeImageIdentity, error) { + trimmed := strings.TrimSpace(value) + if trimmed == "" { + return nil, nil + } + var identities []composeImageIdentity + if strings.HasPrefix(trimmed, "[") { + if json.Unmarshal([]byte(trimmed), &identities) != nil { + return nil, errors.New("Docker Compose returned invalid image identities") + } + } else { + decoder := json.NewDecoder(strings.NewReader(trimmed)) + for { + var item composeImageIdentity + err := decoder.Decode(&item) + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return nil, errors.New("Docker Compose returned invalid image identities") + } + identities = append(identities, item) + } + } + for _, item := range identities { + if item.ID == "" || item.Service == "" && item.ContainerName == "" { + return nil, errors.New("Docker Compose returned invalid image identities") + } + } + return identities, nil +} + func installationRunning(ctx context.Context, installation config.Installation, runner archiveRunner) (bool, error) { result, err := runner.Run(ctx, installation.ComposeArgs("ps", "--all", "--format", "json"), nil) if err != nil { diff --git a/tools/tht/internal/backup/create_test.go b/tools/tht/internal/backup/create_test.go index 5ea50a3e..2d9bd9f1 100644 --- a/tools/tht/internal/backup/create_test.go +++ b/tools/tht/internal/backup/create_test.go @@ -23,6 +23,32 @@ import ( var requiredTestVolumes = []string{"settings", "pi-state", "workspace-registry", "workspace-secrets", "sessions", "qdrant-data", "embedding-models"} +func TestDecodeComposeImageIdentitiesAcceptsArrayAndStreamingJSON(t *testing.T) { + for name, input := range map[string]string{ + "array": `[{"ContainerName":"project-core-1","ID":"sha256:core"},{"ContainerName":"project-frontend-1","ID":"sha256:frontend"}]`, + "streaming": "{\"Service\":\"core\",\"ID\":\"sha256:core\"}\n{\"Service\":\"frontend\",\"ID\":\"sha256:frontend\"}\n", + } { + t.Run(name, func(t *testing.T) { + identities, err := decodeComposeImageIdentities(input) + if err != nil { + t.Fatal(err) + } + if len(identities) != 2 || identities[0].ID != "sha256:core" || identities[1].ID != "sha256:frontend" { + t.Fatalf("image identities = %#v", identities) + } + }) + } +} + +func TestDecodeComposeImageIdentitiesAcceptsNoContainerForProfiledService(t *testing.T) { + for _, input := range []string{"", "[]"} { + identities, err := decodeComposeImageIdentities(input) + if err != nil || len(identities) != 0 { + t.Fatalf("decodeComposeImageIdentities(%q) = %#v, %v", input, identities, err) + } + } +} + func TestCreateWritesManifestLastWithConfigurationMetadataAndSevenVolumes(t *testing.T) { fixture := newBackupFixture(t, "local") output := filepath.Join(t.TempDir(), "custom.zip") diff --git a/tools/tht/internal/backup/preflight.go b/tools/tht/internal/backup/preflight.go index bb8fa20d..e5658b26 100644 --- a/tools/tht/internal/backup/preflight.go +++ b/tools/tht/internal/backup/preflight.go @@ -61,13 +61,15 @@ type PreflightDependencies struct { // ArchiveEntryMetadata is safe restore metadata. It intentionally contains no archive payload. type ArchiveEntryMetadata struct { - Path string - Kind string - Owner string - Size int64 - SHA256 string - Mode uint32 - Sensitive bool + Path string + Kind string + Owner string + LogicalName string + SourcePath string + Size int64 + SHA256 string + Mode uint32 + Sensitive bool } // PreflightResult is the validated, non-mutating input for a future restore transaction. @@ -534,6 +536,8 @@ func reconcileArchiveEntries(ctx context.Context, manifest Manifest, entries map requiredBytes += uint64(actual.metadata.Size) actual.metadata.Kind = entry.Kind actual.metadata.Owner = entry.Owner + actual.metadata.LogicalName = entry.LogicalName + actual.metadata.SourcePath = entry.SourcePath actual.metadata.Sensitive = entry.Sensitive metadata = append(metadata, actual.metadata) delete(entries, entry.Path) diff --git a/tools/tht/internal/backup/preflight_test.go b/tools/tht/internal/backup/preflight_test.go index b2672ea3..8b9f6f18 100644 --- a/tools/tht/internal/backup/preflight_test.go +++ b/tools/tht/internal/backup/preflight_test.go @@ -325,6 +325,7 @@ type preflightArchiveSpec struct { entries []preflightArchiveEntry rawEntries []preflightRawArchiveEntry rawManifest []byte + volumes []VolumeMetadata } type preflightArchiveEntry struct { @@ -337,6 +338,9 @@ type preflightArchiveEntry struct { mode os.FileMode manifestMode *uint32 method uint16 + owner string + logicalName string + sourcePath string } type preflightRawArchiveEntry struct { @@ -374,6 +378,7 @@ func writePreflightArchive(t *testing.T, archivePath string, spec preflightArchi SchemaVersion: spec.schemaVersion, InstallationID: spec.installationID, CreatedAt: time.Date(2026, 8, 16, 10, 0, 0, 0, time.UTC), SourceRevision: testRevision, IncludesSecrets: spec.includeSecrets, ComposeProject: "thothii-test", + Volumes: append([]VolumeMetadata(nil), spec.volumes...), } for _, entry := range spec.entries { checksum := entry.checksum @@ -384,11 +389,18 @@ func writePreflightArchive(t *testing.T, archivePath string, spec preflightArchi if kind == "" { kind = EntryFile } - sourcePath := "" - owner := "installation" + sourcePath := entry.sourcePath + owner := entry.owner + if owner == "" { + owner = "installation" + } if kind == EntryExternalSecret { - sourcePath = "/protected/secret" - owner = "external-secret" + if sourcePath == "" { + sourcePath = "/protected/secret" + } + if entry.owner == "" { + owner = "external-secret" + } } mode := uint32(entry.mode.Perm()) if mode == 0 { @@ -397,7 +409,7 @@ func writePreflightArchive(t *testing.T, archivePath string, spec preflightArchi if entry.manifestMode != nil { mode = *entry.manifestMode } - manifest.Entries = append(manifest.Entries, Entry{Path: entry.path, Kind: kind, Owner: owner, SourcePath: sourcePath, SHA256: checksum, Size: int64(len(entry.body)), Mode: mode, Archived: true, Sensitive: entry.sensitive}) + manifest.Entries = append(manifest.Entries, Entry{Path: entry.path, Kind: kind, Owner: owner, LogicalName: entry.logicalName, SourcePath: sourcePath, SHA256: checksum, Size: int64(len(entry.body)), Mode: mode, Archived: true, Sensitive: entry.sensitive}) } manifestBytes, err := manifest.JSON() if err != nil { diff --git a/tools/tht/internal/backup/restore.go b/tools/tht/internal/backup/restore.go index 609d8887..ffca69de 100644 --- a/tools/tht/internal/backup/restore.go +++ b/tools/tht/internal/backup/restore.go @@ -42,10 +42,11 @@ type restoreDependencies struct { verify map[string]restoreVerify } -// Restore runs the host transaction. Concrete host dependencies are intentionally kept outside -// the deterministic core so callers cannot bypass its preflight and checkpoint boundaries. +// Restore runs the host transaction through the same concrete Docker/filesystem boundaries used +// by backup creation. The injectable core below exists only to make every failure boundary +// deterministic in tests. func Restore(ctx context.Context, installation config.Installation, request RestoreRequest) (RestoreResult, error) { - return RestoreResult{}, errors.New("restore host dependencies are unavailable") + return restoreWithDependencies(ctx, installation, request, productionRestoreDependencies(installation)) } func restoreWithDependencies(ctx context.Context, installation config.Installation, request RestoreRequest, deps restoreDependencies) (result RestoreResult, resultErr error) { @@ -113,9 +114,6 @@ func restoreWithDependencies(ctx context.Context, installation config.Installati members[member.Name] = member } for _, entry := range preflight.Entries { - if entry.Kind == EntryVolume { - continue - } member := members[entry.Path] if member == nil { return result, fmt.Errorf("verified archive is missing %q", entry.Path) @@ -125,7 +123,17 @@ func restoreWithDependencies(ctx context.Context, installation config.Installati return result, fmt.Errorf("open verified archive member %q: %w", entry.Path, openErr) } mutated = true - restoreErr := deps.restoreFile(ctx, installation, entry, stream) + var restoreErr error + if entry.Kind == EntryVolume { + volume, found := restoreVolumeMetadata(preflight.Manifest, entry.LogicalName) + if !found { + _ = stream.Close() + return result, errors.New("verified volume metadata is incomplete") + } + restoreErr = deps.restoreVolume(ctx, installation, volume, stream) + } else { + restoreErr = deps.restoreFile(ctx, installation, entry, stream) + } closeErr := stream.Close() if restoreErr != nil { return result, restoreErr @@ -156,13 +164,25 @@ func restoreWithDependencies(ctx context.Context, installation config.Installati return result, nil } +func restoreVolumeMetadata(manifest Manifest, logicalName string) (VolumeMetadata, bool) { + if logicalName == "" { + return VolumeMetadata{}, false + } + for _, volume := range manifest.Volumes { + if volume.LogicalName == logicalName { + return volume, true + } + } + return VolumeMetadata{}, false +} + // resetAuthenticationState clears browser sessions and pending OIDC transactions without touching // installation-global auth.yaml or users.yaml. The command runs as the unprivileged core user so // the recreated state root is private to the service on both the local volume and server /data bind. func resetAuthenticationState(ctx context.Context, installation config.Installation, runner archiveRunner) error { result, err := runner.Run(ctx, installation.ComposeArgs( "run", "--rm", "--no-deps", "--no-TTY", "--entrypoint", "sh", "core", "-ceu", - "rm -rf /data/auth && install -d -m 0700 /data/auth /data/auth/sessions /data/auth/oidc", + "find /data/auth -mindepth 1 -maxdepth 1 -exec rm -rf -- {} + && install -d -m 0700 /data/auth /data/auth/sessions /data/auth/oidc && test -z \"$(find /data/auth/sessions /data/auth/oidc -mindepth 1 -print -quit)\"", ), nil) if err != nil { return dockerError("reset authentication state", result, err) diff --git a/tools/tht/internal/backup/restore_disk_unix.go b/tools/tht/internal/backup/restore_disk_unix.go new file mode 100644 index 00000000..4236076e --- /dev/null +++ b/tools/tht/internal/backup/restore_disk_unix.go @@ -0,0 +1,22 @@ +//go:build !windows + +package backup + +import ( + "errors" + + "golang.org/x/sys/unix" +) + +func restoreFreeBytes(target string) (uint64, error) { + var statistics unix.Statfs_t + if err := unix.Statfs(target, &statistics); err != nil { + return 0, errors.New("restore filesystem capacity is unavailable") + } + blockSize := uint64(statistics.Bsize) + available := uint64(statistics.Bavail) + if blockSize != 0 && available > ^uint64(0)/blockSize { + return 0, errors.New("restore filesystem capacity is invalid") + } + return blockSize * available, nil +} diff --git a/tools/tht/internal/backup/restore_disk_windows.go b/tools/tht/internal/backup/restore_disk_windows.go new file mode 100644 index 00000000..b9516059 --- /dev/null +++ b/tools/tht/internal/backup/restore_disk_windows.go @@ -0,0 +1,21 @@ +//go:build windows + +package backup + +import ( + "errors" + + "golang.org/x/sys/windows" +) + +func restoreFreeBytes(target string) (uint64, error) { + path, err := windows.UTF16PtrFromString(target) + if err != nil { + return 0, errors.New("restore filesystem capacity is unavailable") + } + var available uint64 + if err := windows.GetDiskFreeSpaceEx(path, &available, nil, nil); err != nil { + return 0, errors.New("restore filesystem capacity is unavailable") + } + return available, nil +} diff --git a/tools/tht/internal/backup/restore_file_unix.go b/tools/tht/internal/backup/restore_file_unix.go new file mode 100644 index 00000000..310117d8 --- /dev/null +++ b/tools/tht/internal/backup/restore_file_unix.go @@ -0,0 +1,121 @@ +//go:build !windows + +package backup + +import ( + "crypto/rand" + "encoding/hex" + "errors" + "io" + "os" + "path/filepath" + "strings" + + "github.com/aritmolab/thothii/tools/tht/internal/safeio" + "golang.org/x/sys/unix" +) + +type restoreTargetIdentity struct { + exists bool + device uint64 + inode uint64 +} + +func replaceRestoreFile(target string, contents []byte, mode os.FileMode) error { + if safeio.ValidateCanonicalPath(target) != nil || mode&os.ModeType != 0 || mode.Perm() == 0 { + return safeio.ErrUnsafeFile + } + components := strings.Split(strings.TrimPrefix(target, string(os.PathSeparator)), string(os.PathSeparator)) + if len(components) < 2 || components[0] == "" || components[len(components)-1] == "" { + return safeio.ErrUnsafeFile + } + directory, err := unix.Open(string(os.PathSeparator), unix.O_RDONLY|unix.O_CLOEXEC|unix.O_DIRECTORY, 0) + if err != nil { + return safeio.ErrUnsafeFile + } + defer unix.Close(directory) + for _, component := range components[:len(components)-1] { + next, openErr := unix.Openat(directory, component, unix.O_RDONLY|unix.O_CLOEXEC|unix.O_DIRECTORY|unix.O_NOFOLLOW, 0) + if openErr != nil { + return safeio.ErrUnsafeFile + } + unix.Close(directory) + directory = next + } + name := components[len(components)-1] + identity, err := inspectRestoreTargetAt(directory, name) + if err != nil { + return safeio.ErrUnsafeFile + } + temporary, err := writeRestoreTemporaryAt(directory, contents, mode.Perm()) + if err != nil { + return safeio.ErrUnsafeFile + } + defer func() { _ = unix.Unlinkat(directory, temporary, 0) }() + current, err := inspectRestoreTargetAt(directory, name) + if err != nil || current != identity { + return safeio.ErrUnsafeFile + } + if err := unix.Renameat(directory, temporary, directory, name); err != nil { + return safeio.ErrUnsafeFile + } + temporary = "" + if err := unix.Fsync(directory); err != nil { + return safeio.ErrUnsafeFile + } + return nil +} + +func inspectRestoreTargetAt(directory int, name string) (restoreTargetIdentity, error) { + var status unix.Stat_t + err := unix.Fstatat(directory, name, &status, unix.AT_SYMLINK_NOFOLLOW) + if errors.Is(err, unix.ENOENT) { + return restoreTargetIdentity{}, nil + } + if err != nil || status.Mode&unix.S_IFMT != unix.S_IFREG || status.Nlink != 1 { + return restoreTargetIdentity{}, safeio.ErrUnsafeFile + } + return restoreTargetIdentity{exists: true, device: uint64(status.Dev), inode: status.Ino}, nil +} + +func writeRestoreTemporaryAt(directory int, contents []byte, mode os.FileMode) (string, error) { + for attempt := 0; attempt < 16; attempt++ { + random := make([]byte, 8) + if _, err := rand.Read(random); err != nil { + return "", err + } + name := ".tht-restore-" + hex.EncodeToString(random) + ".tmp" + descriptor, err := unix.Openat(directory, name, unix.O_WRONLY|unix.O_CREAT|unix.O_EXCL|unix.O_CLOEXEC|unix.O_NOFOLLOW, uint32(mode)) + if errors.Is(err, unix.EEXIST) { + continue + } + if err != nil { + return "", err + } + file := os.NewFile(uintptr(descriptor), filepath.Base(name)) + if file == nil { + unix.Close(descriptor) + return "", safeio.ErrUnsafeFile + } + if err := file.Chmod(mode); err == nil { + var written int + written, err = file.Write(contents) + if err == nil && written != len(contents) { + err = io.ErrShortWrite + } + } + if err == nil { + err = file.Sync() + } + closeErr := file.Close() + if err == nil { + err = closeErr + } + if err != nil { + _ = unix.Unlinkat(directory, name, 0) + return "", err + } + return name, nil + } + return "", safeio.ErrUnsafeFile +} diff --git a/tools/tht/internal/backup/restore_file_windows.go b/tools/tht/internal/backup/restore_file_windows.go new file mode 100644 index 00000000..71572789 --- /dev/null +++ b/tools/tht/internal/backup/restore_file_windows.go @@ -0,0 +1,59 @@ +//go:build windows + +package backup + +import ( + "errors" + "os" + "path/filepath" + + "github.com/aritmolab/thothii/tools/tht/internal/safeio" + "golang.org/x/sys/windows" +) + +func replaceRestoreFile(target string, contents []byte, mode os.FileMode) error { + if safeio.ValidateCanonicalPath(target) != nil || mode&os.ModeType != 0 || mode.Perm() == 0 { + return safeio.ErrUnsafeFile + } + parent := filepath.Dir(target) + resolved, err := filepath.EvalSymlinks(parent) + if err != nil || resolved != parent || !safeWindowsRestoreTarget(target) { + return safeio.ErrUnsafeFile + } + temporary, err := os.CreateTemp(parent, ".tht-restore-*.tmp") + if err != nil { + return safeio.ErrUnsafeFile + } + temporaryPath := temporary.Name() + defer os.Remove(temporaryPath) + if err := temporary.Chmod(mode.Perm()); err == nil { + _, err = temporary.Write(contents) + } + if err == nil { + err = temporary.Sync() + } + closeErr := temporary.Close() + if err == nil { + err = closeErr + } + if err != nil || !safeWindowsRestoreTarget(target) { + return safeio.ErrUnsafeFile + } + from, fromErr := windows.UTF16PtrFromString(temporaryPath) + to, toErr := windows.UTF16PtrFromString(target) + if fromErr != nil || toErr != nil { + return safeio.ErrUnsafeFile + } + if err := windows.MoveFileEx(from, to, windows.MOVEFILE_REPLACE_EXISTING|windows.MOVEFILE_WRITE_THROUGH); err != nil { + return safeio.ErrUnsafeFile + } + return nil +} + +func safeWindowsRestoreTarget(target string) bool { + info, err := os.Lstat(target) + if errors.Is(err, os.ErrNotExist) { + return true + } + return err == nil && info.Mode().IsRegular() && info.Mode()&os.ModeSymlink == 0 +} diff --git a/tools/tht/internal/backup/restore_host.go b/tools/tht/internal/backup/restore_host.go new file mode 100644 index 00000000..495161b3 --- /dev/null +++ b/tools/tht/internal/backup/restore_host.go @@ -0,0 +1,352 @@ +package backup + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/aritmolab/thothii/tools/tht/internal/compose" + "github.com/aritmolab/thothii/tools/tht/internal/config" + "github.com/aritmolab/thothii/tools/tht/internal/doctor" + "github.com/aritmolab/thothii/tools/tht/internal/lifecycle" + "github.com/aritmolab/thothii/tools/tht/internal/pi" + "github.com/aritmolab/thothii/tools/tht/internal/safeio" + "github.com/aritmolab/thothii/tools/tht/internal/service" +) + +func productionRestoreDependencies(installation config.Installation) restoreDependencies { + runner := hostRunner{runner: compose.NewRunner(""), binary: "docker", profile: installation.Profile} + return restoreDependencies{ + preflight: func(ctx context.Context, target config.Installation, request PreflightRequest) (PreflightResult, error) { + return Preflight(ctx, target, request, PreflightDependencies{ + FreeBytes: restoreFreeBytes, + CheckOwnershipPermissions: validateRestoreTargets, + CheckVolumeMapping: func(ctx context.Context, target config.Installation, manifest Manifest) error { + return validateRestoreVolumes(ctx, target, manifest, runner) + }, + CheckImageConfigCompatibility: func(ctx context.Context, target config.Installation, manifest Manifest) error { + return validateRestoreImages(ctx, target, manifest, runner) + }, + }) + }, + checkpoint: func(ctx context.Context, target config.Installation, request CreateRequest) (Result, error) { + path, err := restoreCheckpointPath(target, time.Now().UTC()) + if err != nil { + return Result{}, err + } + request.Output = path + return Create(ctx, target, request) + }, + acquireLock: func(target config.Installation) (restoreLock, error) { + return lifecycle.Acquire(target) + }, + runner: runner, + sleep: time.Sleep, + restoreFile: restoreFilePayload, + restoreVolume: func(ctx context.Context, _ config.Installation, volume VolumeMetadata, input io.Reader) error { + result, err := runner.Stream(ctx, volumeRestoreCommand(volume.Name), input, io.Discard) + if err != nil || result.ExitCode != 0 { + if err == nil { + err = errors.New("Docker volume helper returned a nonzero exit status") + } + return fmt.Errorf("restore volume %s: %w", volume.LogicalName, dockerError("stream volume", result, err)) + } + return nil + }, + resetAuthenticationState: resetAuthenticationState, + verify: map[string]restoreVerify{ + "health": verifyRestoreHealth, + "doctor": verifyRestoreDoctor, + "pi": verifyRestorePi, + "workspace": verifyRestoreWorkspace, + }, + } +} + +func restoreCheckpointPath(installation config.Installation, now time.Time) (string, error) { + suffix := make([]byte, 8) + if _, err := rand.Read(suffix); err != nil { + return "", errors.New("recovery checkpoint name is unavailable") + } + name := fmt.Sprintf("restore-checkpoint-%s-%s.zip", now.Format("20060102T150405.000000000Z"), hex.EncodeToString(suffix)) + return filepath.Join(installation.ControlDirectory(), name), nil +} + +func validateRestoreTargets(_ context.Context, installation config.Installation, manifest Manifest) error { + for _, entry := range manifest.Entries { + if entry.Kind == EntryVolume { + continue + } + if !entry.Archived { + if entry.Kind == EntrySecretReference || entry.Kind == EntryPreservationReference { + if err := validateRestoreReference(installation, entry); err != nil { + return err + } + } + continue + } + metadata := ArchiveEntryMetadata{ + Path: entry.Path, Kind: entry.Kind, Owner: entry.Owner, LogicalName: entry.LogicalName, + SourcePath: entry.SourcePath, Size: entry.Size, SHA256: entry.SHA256, Mode: entry.Mode, + Sensitive: entry.Sensitive, + } + target, err := restoreFileTarget(installation, metadata) + if err != nil || !safeRestoreParent(target) { + return errors.New("restore target ownership or permissions are invalid") + } + } + return nil +} + +func validateRestoreReference(installation config.Installation, entry Entry) error { + metadata := ArchiveEntryMetadata{ + Path: entry.Path, Kind: entry.Kind, Owner: entry.Owner, SourcePath: entry.SourcePath, + Size: entry.Size, SHA256: entry.SHA256, Mode: entry.Mode, Sensitive: entry.Sensitive, + } + if entry.Kind == EntryPreservationReference { + return nil + } + if _, err := restoreExternalTarget(installation, metadata); err != nil { + return errors.New("restore external prerequisite is invalid") + } + contents, err := safeio.ReadCanonicalRegular(entry.SourcePath, entry.Size) + if err != nil || int64(len(contents)) != entry.Size { + return errors.New("restore external prerequisite is unavailable or unsafe") + } + digest := sha256.Sum256(contents) + if "sha256:"+hex.EncodeToString(digest[:]) != entry.SHA256 { + return errors.New("restore external prerequisite has changed") + } + return nil +} + +func validateRestoreVolumes(ctx context.Context, installation config.Installation, manifest Manifest, runner archiveRunner) error { + if len(manifest.Volumes) != len(requiredVolumes) { + return errors.New("backup volume set is incomplete") + } + rendered, err := renderedConfiguration(ctx, installation, runner) + if err != nil { + return err + } + current, err := inspectRequiredVolumes(ctx, runner, rendered) + if err != nil { + return err + } + archived := make(map[string]VolumeMetadata, len(manifest.Volumes)) + for _, volume := range manifest.Volumes { + archived[volume.LogicalName] = volume + } + for _, volume := range current { + previous, found := archived[volume.LogicalName] + if !found || previous.Name != volume.Name || previous.Driver != volume.Driver { + return errors.New("backup volume ownership does not match the installation") + } + if volume.Labels["com.docker.compose.project"] != installation.ProjectName() { + return errors.New("current volume is not owned by the installation") + } + } + return nil +} + +func validateRestoreImages(ctx context.Context, installation config.Installation, manifest Manifest, runner archiveRunner) error { + rendered, err := renderedConfiguration(ctx, installation, runner) + if err != nil { + return err + } + for _, image := range manifest.Images { + serviceDefinition, found := rendered.Services[image.Service] + if !found || serviceDefinition.Image == "" || serviceDefinition.Image != image.Reference { + return errors.New("backup image configuration does not match the installation") + } + } + return nil +} + +func restoreFilePayload(_ context.Context, installation config.Installation, entry ArchiveEntryMetadata, input io.Reader) error { + if entry.Size < 0 || uint64(entry.Size) > defaultPreflightMaxUncompressedBytes { + return errors.New("restore file size is invalid") + } + contents, err := io.ReadAll(io.LimitReader(input, entry.Size+1)) + if err != nil || int64(len(contents)) != entry.Size { + return errors.New("restore file payload is invalid") + } + target, err := restoreFileTarget(installation, entry) + if err != nil { + return err + } + if err := replaceRestoreFile(target, contents, os.FileMode(entry.Mode)); err != nil { + return errors.New("restore file could not be replaced safely") + } + return nil +} + +func restoreFileTarget(installation config.Installation, entry ArchiveEntryMetadata) (string, error) { + if entry.Kind == EntryExternalSecret { + return restoreExternalTarget(installation, entry) + } + if entry.Kind != EntryFile { + return "", errors.New("restore file kind is unsupported") + } + switch entry.Path { + case "configuration/installation/thothii-installation.yaml": + return installation.Path, nil + case "configuration/environment/operator.env": + return installation.EnvFile, nil + case "configuration/pi/models.json": + return filepath.Join(installation.ProjectDirectory, "deploy", "pi", "models.json"), nil + case "configuration/pi/settings.json": + return filepath.Join(installation.ProjectDirectory, "deploy", "pi", "settings.json"), nil + case "configuration/generated/current-image.yaml": + return installation.CurrentImageOverridePath(), nil + } + if strings.HasPrefix(entry.Path, "configuration/overrides/") { + name := strings.TrimPrefix(entry.Path, "configuration/overrides/") + indexText, base, found := strings.Cut(name, "-") + index, parseErr := strconv.Atoi(indexText) + if !found || parseErr != nil || len(indexText) != 2 || index < 0 || index >= len(installation.Overrides) || filepath.Base(installation.Overrides[index]) != base { + return "", errors.New("restore override target is invalid") + } + return installation.Overrides[index], nil + } + if strings.HasPrefix(entry.Owner, "preservation-root:") && strings.HasPrefix(entry.Path, "preservation/") { + variable := strings.TrimPrefix(entry.Owner, "preservation-root:") + allowed := variable == "THT_DATA_ROOT" || variable == "THT_PI_STATE_ROOT" || variable == "THT_WORKSPACE_REGISTRY_ROOT" + parts := strings.SplitN(entry.Path, "/", 3) + root, rootErr := installation.EnvironmentValue(variable) + if !allowed || len(parts) != 3 || rootErr != nil || root == "" { + return "", errors.New("restore preservation target is invalid") + } + target := filepath.Join(root, filepath.FromSlash(parts[2])) + relative, relErr := filepath.Rel(root, target) + if relErr != nil || relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) { + return "", errors.New("restore preservation target escapes its root") + } + return target, nil + } + return "", errors.New("restore file target is not declared") +} + +func restoreExternalTarget(installation config.Installation, entry ArchiveEntryMetadata) (string, error) { + if entry.SourcePath == "" || filepath.Clean(entry.SourcePath) != entry.SourcePath || !filepath.IsAbs(entry.SourcePath) { + return "", errors.New("restore external target is invalid") + } + if entry.Owner == "external-secret" { + paths, err := installation.SecretFiles() + if err != nil { + return "", errors.New("restore external secret declarations are unavailable") + } + for _, path := range paths { + if path == entry.SourcePath { + return path, nil + } + } + return "", errors.New("restore external secret target is not declared") + } + if entry.Owner == "authentication-configuration" { + for _, name := range []string{"auth.yaml", "users.yaml"} { + path := filepath.Join(installation.AuthenticationDirectory(), name) + if entry.SourcePath == path { + return path, nil + } + } + } + return "", errors.New("restore external target owner is invalid") +} + +func safeRestoreParent(target string) bool { + if target == "" || !filepath.IsAbs(target) || filepath.Clean(target) != target { + return false + } + parent := filepath.Dir(target) + resolved, err := filepath.EvalSymlinks(parent) + if err != nil || resolved != parent { + return false + } + info, err := os.Stat(parent) + if err != nil || !info.IsDir() { + return false + } + if targetInfo, err := os.Lstat(target); err == nil { + return targetInfo.Mode().IsRegular() && targetInfo.Mode()&os.ModeSymlink == 0 + } else { + return errors.Is(err, os.ErrNotExist) + } +} + +func volumeRestoreCommand(volume string) []string { + return []string{ + "run", "--rm", "--network", "none", "--mount", "type=volume,src=" + volume + ",dst=/target", + helperImage, "sh", "-ceu", + "rm -rf -- /target/* /target/.[!.]* /target/..?*; tar --numeric-owner -C /target -xf -", + } +} + +func restoreVerificationRunning(ctx context.Context, installation config.Installation, runner archiveRunner) (bool, error) { + return installationRunning(ctx, installation, runner) +} + +func verifyRestoreHealth(ctx context.Context, installation config.Installation, runner archiveRunner) error { + running, err := restoreVerificationRunning(ctx, installation, runner) + if err != nil || !running { + return err + } + return service.WaitForHealthy(ctx, installation, runner) +} + +func verifyRestoreDoctor(ctx context.Context, installation config.Installation, runner archiveRunner) error { + running, err := restoreVerificationRunning(ctx, installation, runner) + if err != nil { + return err + } + if !running { + result, configErr := runner.Run(ctx, installation.ComposeArgs("config", "--quiet"), nil) + if configErr != nil { + return dockerError("verify restored Compose configuration", result, configErr) + } + return nil + } + report, err := doctor.Run(ctx, installation, runner) + if err != nil { + return err + } + if !report.OK { + return errors.New("aggregate doctor did not pass after restore") + } + return nil +} + +func verifyRestorePi(ctx context.Context, installation config.Installation, runner archiveRunner) error { + running, err := restoreVerificationRunning(ctx, installation, runner) + if err != nil || !running { + return err + } + return pi.Doctor(ctx, compose.InstallationRunner{Installation: installation, Runner: runner}) +} + +func verifyRestoreWorkspace(ctx context.Context, installation config.Installation, runner archiveRunner) error { + running, err := restoreVerificationRunning(ctx, installation, runner) + if err != nil || !running { + return err + } + result, err := runner.Run(ctx, installation.ComposeArgs( + "exec", "-T", "core", "curl", "-fsS", "--max-time", "5", "http://127.0.0.1:8787/workspaces", + ), nil) + if err != nil { + return dockerError("inspect restored workspaces", result, err) + } + var workspaces []json.RawMessage + if json.Unmarshal([]byte(result.Stdout), &workspaces) != nil { + return errors.New("restored workspace inspection returned invalid JSON") + } + return nil +} diff --git a/tools/tht/internal/backup/restore_test.go b/tools/tht/internal/backup/restore_test.go index 4600f4ed..3f9cde8e 100644 --- a/tools/tht/internal/backup/restore_test.go +++ b/tools/tht/internal/backup/restore_test.go @@ -1,6 +1,8 @@ package backup import ( + "archive/tar" + "bytes" "context" "errors" "io" @@ -13,6 +15,85 @@ import ( "github.com/aritmolab/thothii/tools/tht/internal/config" ) +func TestRestorePublicPathUsesConcreteProductionPreflight(t *testing.T) { + root := t.TempDir() + installation := config.Installation{ + Path: filepath.Join(root, "deploy", "local-dev", "thothii-installation.yaml"), + ProjectDirectory: root, + } + missing := filepath.Join(root, "missing.zip") + + _, err := Restore(context.Background(), installation, RestoreRequest{Archive: missing, Confirm: true}) + + if err == nil || strings.Contains(err.Error(), "dependencies are unavailable") || !strings.Contains(err.Error(), "backup archive") { + t.Fatalf("Restore() error = %v, want production archive preflight", err) + } +} + +func TestRestoreRestoresVerifiedVolumesInManifestOrderBeforeAuthenticationReset(t *testing.T) { + installation := preflightTestInstallation(t) + archive := filepath.Join(t.TempDir(), "restore-volumes.zip") + sessionsTar := safeRestoreTar(t, "session.txt", "session") + settingsTar := safeRestoreTar(t, "settings.json", "settings") + writePreflightArchive(t, archive, preflightArchiveSpec{ + volumes: []VolumeMetadata{ + {LogicalName: "sessions", Name: "project_sessions", Driver: "local"}, + {LogicalName: "settings", Name: "project_settings", Driver: "local"}, + }, + entries: []preflightArchiveEntry{ + {path: "configuration/operator.env", body: []byte("safe")}, + {path: "volumes/settings.tar", body: settingsTar, kind: EntryVolume, owner: "volume:settings", logicalName: "settings"}, + {path: "volumes/sessions.tar", body: sessionsTar, kind: EntryVolume, owner: "volume:sessions", logicalName: "sessions"}, + }, + }) + runner := newBackupRunner(installation, false) + deps := restoreTestDependencies(t, runner) + var events []string + deps.restoreFile = func(_ context.Context, _ config.Installation, entry ArchiveEntryMetadata, _ io.Reader) error { + events = append(events, "file:"+entry.Path) + return nil + } + deps.restoreVolume = func(_ context.Context, _ config.Installation, volume VolumeMetadata, stream io.Reader) error { + if _, err := io.ReadAll(stream); err != nil { + return err + } + events = append(events, "volume:"+volume.LogicalName) + return nil + } + deps.resetAuthenticationState = func(context.Context, config.Installation, archiveRunner) error { + events = append(events, "reset-auth-state") + return nil + } + + if _, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: archive, Confirm: true}, deps); err != nil { + t.Fatal(err) + } + if got, want := events, []string{ + "file:configuration/operator.env", + "volume:sessions", + "volume:settings", + "reset-auth-state", + }; !equalStrings(got, want) { + t.Fatalf("restore events = %v, want %v", got, want) + } +} + +func safeRestoreTar(t *testing.T, name, contents string) []byte { + t.Helper() + var output bytes.Buffer + writer := tar.NewWriter(&output) + if err := writer.WriteHeader(&tar.Header{Name: name, Mode: 0o600, Size: int64(len(contents)), Typeflag: tar.TypeReg}); err != nil { + t.Fatal(err) + } + if _, err := writer.Write([]byte(contents)); err != nil { + t.Fatal(err) + } + if err := writer.Close(); err != nil { + t.Fatal(err) + } + return output.Bytes() +} + func TestRestoreStoppedInstallationRunsCheckpointRestoreAndVerification(t *testing.T) { installation := preflightTestInstallation(t) archive := filepath.Join(t.TempDir(), "restore.zip") @@ -103,7 +184,9 @@ func TestResetAuthenticationStateCreatesOnlyPrivateEmptyStateDirectories(t *test joined := strings.Join(runner.args, "\x00") for _, required := range []string{ "run", "--rm", "--no-deps", "--no-TTY", "--entrypoint", "sh", "core", "-ceu", - "rm -rf /data/auth && install -d -m 0700 /data/auth /data/auth/sessions /data/auth/oidc", + "find /data/auth -mindepth 1 -maxdepth 1 -exec rm -rf -- {} +", + "install -d -m 0700 /data/auth /data/auth/sessions /data/auth/oidc", + "find /data/auth/sessions /data/auth/oidc -mindepth 1 -print -quit", } { if !strings.Contains(joined, required) { t.Fatalf("authentication state reset command omits %q: %#v", required, runner.args)