fix(auth): validate stopped workspace restore
This commit is contained in:
@@ -18,7 +18,8 @@ import { WorkspaceRegistry } from "./workspaces/registry.js";
|
||||
import { WorkspaceSecretStore } from "./workspaces/secret-store.js";
|
||||
|
||||
type OperatorAction = "maintenance-activate" | "maintenance-deactivate" | "maintenance-status"
|
||||
| "session-inventory" | "workflow-doctor" | "pi-options" | "pi-test" | "effective-settings";
|
||||
| "session-inventory" | "workflow-doctor" | "workspace-integrity"
|
||||
| "pi-options" | "pi-test" | "effective-settings";
|
||||
|
||||
const lifecyclePrincipal: PrincipalContext = {
|
||||
issuer: "tht-operator-command",
|
||||
@@ -86,6 +87,15 @@ async function workflowDiagnostics(config: AppConfig): Promise<{ ready: true; wo
|
||||
return { ready: true, workspaces: revisions.length };
|
||||
}
|
||||
|
||||
async function workspaceIntegrity(config: AppConfig): Promise<{
|
||||
ready: true;
|
||||
state: "uninitialized" | "active";
|
||||
workspaces: number;
|
||||
}> {
|
||||
const integrity = await new WorkspaceRegistry(config.workspaceRegistry).verifyStoredState();
|
||||
return { ready: true, ...integrity };
|
||||
}
|
||||
|
||||
export async function runOperatorAction(
|
||||
action: OperatorAction,
|
||||
config: AppConfig,
|
||||
@@ -98,6 +108,7 @@ export async function runOperatorAction(
|
||||
}
|
||||
if (action === "session-inventory") return await sessionInventory(config);
|
||||
if (action === "workflow-doctor") return await workflowDiagnostics(config);
|
||||
if (action === "workspace-integrity") return await workspaceIntegrity(config);
|
||||
if (action === "effective-settings") return effectiveSettings(config, loadSettings(config));
|
||||
const service = createPiManagement(config, { listModels: createPiModelLister(config) });
|
||||
if (action === "pi-options") return await service.options();
|
||||
@@ -109,7 +120,7 @@ async function main(): Promise<void> {
|
||||
const action = process.argv[2] as OperatorAction | undefined;
|
||||
if (!action || ![
|
||||
"maintenance-activate", "maintenance-deactivate", "maintenance-status", "session-inventory",
|
||||
"workflow-doctor", "pi-options", "pi-test", "effective-settings",
|
||||
"workflow-doctor", "workspace-integrity", "pi-options", "pi-test", "effective-settings",
|
||||
].includes(action)) throw new Error("invalid operator action");
|
||||
const result = await runOperatorAction(action, loadConfig(process.env));
|
||||
process.stdout.write(`${JSON.stringify(result)}\n`);
|
||||
|
||||
@@ -31,6 +31,11 @@ export interface WorkspaceRevision {
|
||||
snapshotPath: string;
|
||||
}
|
||||
|
||||
export interface StoredWorkspaceIntegrity {
|
||||
state: "uninitialized" | "active";
|
||||
workspaces: number;
|
||||
}
|
||||
|
||||
export interface SessionRevisionLease {
|
||||
workspace: WorkspaceDescriptor;
|
||||
revision: WorkspaceRevision;
|
||||
@@ -180,6 +185,48 @@ export class WorkspaceRegistry {
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate only persisted local registry state. Restore uses this path while the installation
|
||||
* is stopped: it must neither contact Git nor turn a never-used registry into initialized state.
|
||||
*/
|
||||
async verifyStoredState(): Promise<StoredWorkspaceIntegrity> {
|
||||
await this.repository.ensureLayout();
|
||||
return await this.lock.run(async () => {
|
||||
try {
|
||||
const stateEntries = await readdir(this.repository.statePath, { withFileTypes: true });
|
||||
if (stateEntries.some((entry) => (
|
||||
entry.name !== "active.json" || !entry.isFile() || entry.isSymbolicLink()
|
||||
))) throw new Error("workspace state directory is partial");
|
||||
|
||||
const snapshotEntries = await readdir(this.repository.snapshotsPath, { withFileTypes: true });
|
||||
for (const entry of snapshotEntries) {
|
||||
const isRuntime = entry.name === "runtime";
|
||||
const isSnapshot = /^[0-9a-f]{40}$/.test(entry.name);
|
||||
if ((!isRuntime && !isSnapshot) || !entry.isDirectory() || entry.isSymbolicLink()) {
|
||||
throw new Error("workspace snapshot directory is partial");
|
||||
}
|
||||
}
|
||||
|
||||
const hasActiveState = stateEntries.length === 1;
|
||||
if (!hasActiveState) {
|
||||
if (snapshotEntries.some((entry) => entry.name !== "runtime") || this.storedPathExists(this.repository.repoPath)) {
|
||||
throw new Error("workspace registry is partially initialized");
|
||||
}
|
||||
return { state: "uninitialized", workspaces: 0 };
|
||||
}
|
||||
|
||||
const active = await this.activeState();
|
||||
for (const entry of snapshotEntries) {
|
||||
if (entry.name === "runtime" || entry.name === active.head) continue;
|
||||
await this.snapshotState(entry.name);
|
||||
}
|
||||
return { state: "active", workspaces: active.revisions.length };
|
||||
} catch (error) {
|
||||
throw workspaceError(error);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async read(id: string): Promise<{ workspace: WorkspaceDescriptor; revision: WorkspaceRevision }> {
|
||||
const state = await this.activeState();
|
||||
const revision = state.revisions.find((candidate) => candidate.id === id);
|
||||
@@ -737,6 +784,20 @@ export class WorkspaceRegistry {
|
||||
}
|
||||
}
|
||||
|
||||
private storedPathExists(path: string): boolean {
|
||||
try {
|
||||
const entry = lstatSync(path);
|
||||
if (!entry.isDirectory() || entry.isSymbolicLink()) {
|
||||
throw new WorkspaceRegistryError("workspace_invalid", "Workspace registry path is invalid");
|
||||
}
|
||||
return true;
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code === "ENOENT") return false;
|
||||
if (error instanceof WorkspaceRegistryError) throw error;
|
||||
throw new WorkspaceRegistryError("workspace_invalid", "Workspace registry path is unavailable");
|
||||
}
|
||||
}
|
||||
|
||||
private pathIsMissing(path: string): boolean {
|
||||
try {
|
||||
lstatSync(path);
|
||||
|
||||
@@ -338,6 +338,41 @@ function persistedState(root: string, commit: string): { active: any; manifest:
|
||||
};
|
||||
}
|
||||
|
||||
test("verifies a never-initialized registry without contacting its remote", async () => {
|
||||
const root = mkdtempSync(join(tmpdir(), "thoth-workspace-registry-uninitialized-"));
|
||||
temporaryRoots.push(root);
|
||||
const registryRoot = join(root, "registry");
|
||||
const registry = new WorkspaceRegistry(config(
|
||||
registryRoot,
|
||||
join(root, "missing-remote.git"),
|
||||
));
|
||||
|
||||
await expect(registry.verifyStoredState()).resolves.toEqual({
|
||||
state: "uninitialized",
|
||||
workspaces: 0,
|
||||
});
|
||||
expect(existsSync(join(registryRoot, "repo"))).toBe(false);
|
||||
expect(readdirSync(join(registryRoot, "state"))).toEqual([]);
|
||||
});
|
||||
|
||||
test("verifies initialized snapshots and rejects partial or malformed persisted state", async () => {
|
||||
const remote = await fixture();
|
||||
const registryRoot = join(remote.root, "registry");
|
||||
const registry = new WorkspaceRegistry(config(registryRoot, remote.remote));
|
||||
await registry.bootstrap();
|
||||
|
||||
await expect(registry.verifyStoredState()).resolves.toEqual({
|
||||
state: "active",
|
||||
workspaces: 1,
|
||||
});
|
||||
|
||||
rmSync(join(registryRoot, "state", "active.json"));
|
||||
await expect(registry.verifyStoredState()).rejects.toMatchObject({ code: "workspace_invalid" });
|
||||
|
||||
writeFileSync(join(registryRoot, "state", "active.json"), "{malformed");
|
||||
await expect(registry.verifyStoredState()).rejects.toMatchObject({ code: "workspace_invalid" });
|
||||
});
|
||||
|
||||
test("rejects a catalog entry without a descriptor instead of creating a bootstrap slot", async () => {
|
||||
const remote = await contentOnlyFixture();
|
||||
const registry = new WorkspaceRegistry(config(join(remote.root, "registry"), remote.remote));
|
||||
|
||||
@@ -983,7 +983,8 @@ PY
|
||||
}
|
||||
|
||||
task13_assert_server_oidc_restore_verification() {
|
||||
local archive frontend status diagnostics
|
||||
local archive frontend status diagnostics active_state_before restore_output restore_rc restore_cause
|
||||
local rollback_output rollback_rc rollback_sentinel provider_label checkpoint_leftover
|
||||
archive="$TASK13_TMP/server-oidc-restore-source.zip"
|
||||
frontend="$(task13_frontend_address)"
|
||||
task13_compose_logged "seed valid server authentication runtime excluded from restore" exec -T core node --input-type=module -e '
|
||||
@@ -1003,14 +1004,88 @@ task13_assert_server_oidc_restore_verification() {
|
||||
browserTransactionDigest: "d".repeat(64), browserTransactionTransport: "https",
|
||||
});
|
||||
'
|
||||
active_state_before="$(task13_compose exec -T core node -e '
|
||||
const fs = require("node:fs");
|
||||
process.stdout.write(fs.existsSync("/data/workspace-registry/state/active.json") ? "present" : "absent");
|
||||
')"
|
||||
[[ "$active_state_before" == present || "$active_state_before" == absent ]] \
|
||||
|| task13_fail "server restore fixture returned an invalid registry state diagnostic"
|
||||
task13_compose_logged "stop server stack for OIDC restore" stop
|
||||
rollback_sentinel="$TASK13_SERVER_DATA/task13-restore-rollback"
|
||||
printf 'backup-state\n' >"$rollback_sentinel"
|
||||
task13_run_logged "create real server default-custody backup" "$TASK13_THT" \
|
||||
--installation "$TASK13_INSTALLATION" backup --output "$archive"
|
||||
task13_run_logged "perform real stopped OIDC production restore verification" "$TASK13_THT" \
|
||||
--installation "$TASK13_INSTALLATION" restore "$archive" --yes
|
||||
printf 'current-state\n' >"$rollback_sentinel"
|
||||
|
||||
provider_label="$(docker container inspect --format '{{ index .Config.Labels "io.thothii.task13.run" }}' "$TASK13_OIDC_CONTAINER")"
|
||||
[[ "$provider_label" == "$TASK13_RUN_ID" ]] || task13_fail "fake OIDC provider ownership changed before rollback injection"
|
||||
task13_run_logged "inject post-mutation OIDC verification failure" docker stop "$TASK13_OIDC_CONTAINER"
|
||||
rollback_output="$TASK13_TMP/server-oidc-rollback.out"
|
||||
set +e
|
||||
"$TASK13_THT" --installation "$TASK13_INSTALLATION" restore "$archive" --yes \
|
||||
>"$rollback_output" 2>&1
|
||||
rollback_rc=$?
|
||||
set -e
|
||||
[[ "$rollback_rc" -ne 0 ]] || task13_fail "server restore unexpectedly passed with its OIDC provider unavailable"
|
||||
grep -Eq 'restore verification doctor:|authentication diagnostics did not pass after restore' "$rollback_output" \
|
||||
|| task13_fail "server restore rollback injection did not reach post-mutation authentication verification"
|
||||
if grep -Fq "$TASK13_OIDC_CLIENT_SECRET" "$rollback_output" \
|
||||
|| grep -Fq "$TASK13_AUTHENTIK_API_TOKEN" "$rollback_output"; then
|
||||
task13_fail "failed server restore exposed fake-provider custody values"
|
||||
fi
|
||||
[[ "$(cat "$rollback_sentinel")" == current-state ]] \
|
||||
|| task13_fail "failed server restore did not roll back the server data bind"
|
||||
checkpoint_leftover="$(find "$TASK13_CONTROL_DIR" -maxdepth 1 -name 'restore-checkpoint-*.zip' -print -quit)"
|
||||
[[ -z "$checkpoint_leftover" ]] || task13_fail "failed server restore retained its private recovery checkpoint"
|
||||
task13_compose_logged "verify failed restore cleared authentication runtime" \
|
||||
run --rm --no-deps --no-TTY core sh -ceu '
|
||||
test "$(stat -c %a /data/auth)" = 700
|
||||
test "$(stat -c %a /data/auth/sessions)" = 700
|
||||
test "$(stat -c %a /data/auth/oidc)" = 700
|
||||
test "$(stat -c %u /data/auth)" = "$(id -u)"
|
||||
test -z "$(find /data/auth/sessions /data/auth/oidc -mindepth 1 -print -quit)"
|
||||
'
|
||||
task13_run_logged "restore scoped fake OIDC provider after failure injection" docker start "$TASK13_OIDC_CONTAINER"
|
||||
for _attempt in $(seq 1 30); do
|
||||
if docker exec "$TASK13_OIDC_CONTAINER" node -e \
|
||||
"fetch('https://task13-fake-oidc:9443/health',{signal:AbortSignal.timeout(3000)}).then(r=>{if(!r.ok)process.exit(1)}).catch(()=>process.exit(1))" \
|
||||
>>"$TASK13_LOG" 2>&1; then
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
docker exec "$TASK13_OIDC_CONTAINER" node -e \
|
||||
"fetch('https://task13-fake-oidc:9443/health',{signal:AbortSignal.timeout(3000)}).then(r=>{if(!r.ok)process.exit(1)}).catch(()=>process.exit(1))" \
|
||||
>>"$TASK13_LOG" 2>&1 || task13_log_failure "restored scoped fake OIDC provider readiness"
|
||||
printf 'Task 13 server rollback injection passed: exit=%s; recovery checkpoint removed.\n' "$rollback_rc"
|
||||
|
||||
restore_output="$TASK13_TMP/server-oidc-restore.out"
|
||||
set +e
|
||||
"$TASK13_THT" --installation "$TASK13_INSTALLATION" restore "$archive" --yes \
|
||||
>"$restore_output" 2>&1
|
||||
restore_rc=$?
|
||||
set -e
|
||||
if [[ "$restore_rc" -ne 0 ]]; then
|
||||
restore_cause=restore-failed
|
||||
if grep -Fq 'restore verification workspace: validate restored workspace registry' "$restore_output"; then
|
||||
restore_cause=workspace-validator-rejected
|
||||
fi
|
||||
printf 'Task 13 stopped restore diagnostic: exit=%s active-state-before=%s cause=%s\n' \
|
||||
"$restore_rc" "$active_state_before" "$restore_cause" >&2
|
||||
task13_sanitize <"$restore_output" | tail -n 8 >&2
|
||||
return "$restore_rc"
|
||||
fi
|
||||
checkpoint_leftover="$(find "$TASK13_CONTROL_DIR" -maxdepth 1 -name 'restore-checkpoint-*.zip' -print -quit)"
|
||||
[[ -z "$checkpoint_leftover" ]] || task13_fail "successful server restore retained its private recovery checkpoint"
|
||||
task13_compose_start_logged "start restored server stack" up --detach --wait --wait-timeout 120 core frontend
|
||||
status="$(curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" --max-time "$TASK13_CURL_MAX_TIME" \
|
||||
--silent --output /dev/null --write-out '%{http_code}' "http://$frontend/api/me")"
|
||||
frontend="$(task13_frontend_address)"
|
||||
status=""
|
||||
for _attempt in $(seq 1 30); do
|
||||
status="$(curl --connect-timeout "$TASK13_CURL_CONNECT_TIMEOUT" --max-time "$TASK13_CURL_MAX_TIME" \
|
||||
--silent --output /dev/null --write-out '%{http_code}' "http://$frontend/api/me" || true)"
|
||||
[[ "$status" == 401 ]] && break
|
||||
sleep 1
|
||||
done
|
||||
[[ "$status" == 401 ]] || task13_fail "OIDC restore did not require browser reauthentication"
|
||||
task13_compose_logged "verify private empty server authentication state" exec -T core sh -ceu '
|
||||
test "$(stat -c %a /data/auth)" = 700
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
@@ -407,14 +408,35 @@ func verifyRestoreWorkspace(ctx context.Context, installation config.Installatio
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
command := []string{"exec", "-T", "core", "node", "-e"}
|
||||
command := []string{"exec", "-T", "core", "node", "/app/backend/dist/operator-command.js", "workspace-integrity"}
|
||||
if !running {
|
||||
command = []string{"run", "--rm", "--no-deps", "--no-TTY", "core", "node", "-e"}
|
||||
command = []string{"run", "--rm", "--no-deps", "--no-TTY", "core", "node", "/app/backend/dist/operator-command.js", "workspace-integrity"}
|
||||
}
|
||||
command = append(command, `const fs=require("node:fs");const p="/data/workspace-registry/state/active.json";const s=JSON.parse(fs.readFileSync(p,"utf8"));const h=/^[0-9a-f]{40}$/;if(!h.test(s.head)||!Array.isArray(s.revisions)||s.revisions.some(r=>!r||typeof r.id!=="string"||!r.id||!h.test(r.commit)||!h.test(r.blob)))process.exit(1);for(const r of s.revisions)fs.accessSync("/data/workspace-registry/snapshots/"+r.commit+"/"+r.id+".yaml",fs.constants.R_OK)`)
|
||||
result, err := runner.Run(ctx, installation.ComposeArgs(command...), nil)
|
||||
if err != nil {
|
||||
if err != nil || result.ExitCode != 0 {
|
||||
return dockerError("validate restored workspace registry", result, err)
|
||||
}
|
||||
if len(result.Stdout) == 0 || len(result.Stdout) > 4096 {
|
||||
return errors.New("restored workspace registry returned an invalid result")
|
||||
}
|
||||
var payload struct {
|
||||
Ready bool `json:"ready"`
|
||||
State string `json:"state"`
|
||||
Workspaces int `json:"workspaces"`
|
||||
}
|
||||
decoder := json.NewDecoder(strings.NewReader(result.Stdout))
|
||||
decoder.DisallowUnknownFields()
|
||||
if decodeErr := decoder.Decode(&payload); decodeErr != nil {
|
||||
return errors.New("restored workspace registry returned an invalid result")
|
||||
}
|
||||
var trailing any
|
||||
if decodeErr := decoder.Decode(&trailing); !errors.Is(decodeErr, io.EOF) {
|
||||
return errors.New("restored workspace registry returned an invalid result")
|
||||
}
|
||||
if !payload.Ready || payload.Workspaces < 0 ||
|
||||
(payload.State != "active" && payload.State != "uninitialized") ||
|
||||
(payload.State == "uninitialized" && payload.Workspaces != 0) {
|
||||
return errors.New("restored workspace registry did not pass integrity validation")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -511,6 +512,90 @@ func (runner *authenticationStateResetRunner) Stream(context.Context, []string,
|
||||
|
||||
func (*authenticationStateResetRunner) SessionInventoryScope() string { return "mine" }
|
||||
|
||||
type workspaceVerificationRunner struct {
|
||||
running bool
|
||||
result compose.Result
|
||||
err error
|
||||
calls []string
|
||||
}
|
||||
|
||||
func (runner *workspaceVerificationRunner) Run(_ context.Context, args []string, _ io.Reader) (compose.Result, error) {
|
||||
command := strings.Join(args, " ")
|
||||
runner.calls = append(runner.calls, command)
|
||||
if strings.Contains(command, " ps --all --format json") {
|
||||
if runner.running {
|
||||
return compose.Result{Stdout: healthyServicesPayload()}, nil
|
||||
}
|
||||
return compose.Result{}, nil
|
||||
}
|
||||
if strings.Contains(command, "operator-command.js workspace-integrity") {
|
||||
return runner.result, runner.err
|
||||
}
|
||||
return compose.Result{}, fmt.Errorf("unexpected workspace verification command: %s", command)
|
||||
}
|
||||
|
||||
func (*workspaceVerificationRunner) Stream(context.Context, []string, io.Reader, io.Writer) (compose.Result, error) {
|
||||
return compose.Result{}, errors.New("workspace verification must not stream")
|
||||
}
|
||||
|
||||
func (*workspaceVerificationRunner) SessionInventoryScope() string { return "mine" }
|
||||
|
||||
func TestVerifyRestoreWorkspaceUsesFixedNonNetworkOperatorPath(t *testing.T) {
|
||||
installation := preflightTestInstallation(t)
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
running bool
|
||||
payload string
|
||||
prefix string
|
||||
}{
|
||||
{name: "stopped uninitialized", payload: `{"ready":true,"state":"uninitialized","workspaces":0}`, prefix: "run --rm --no-deps --no-TTY core"},
|
||||
{name: "running active", running: true, payload: `{"ready":true,"state":"active","workspaces":1}`, prefix: "exec -T core"},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
runner := &workspaceVerificationRunner{
|
||||
running: test.running,
|
||||
result: compose.Result{Stdout: test.payload},
|
||||
}
|
||||
if err := verifyRestoreWorkspace(context.Background(), installation, runner); err != nil {
|
||||
t.Fatalf("verify restored workspace: %v", err)
|
||||
}
|
||||
if len(runner.calls) != 2 || !strings.Contains(runner.calls[1], test.prefix+" node /app/backend/dist/operator-command.js workspace-integrity") {
|
||||
t.Fatalf("workspace verification calls = %#v", runner.calls)
|
||||
}
|
||||
for _, call := range runner.calls {
|
||||
if strings.Contains(call, "curl") || strings.Contains(call, "-e const") || strings.Contains(strings.ToLower(call), "header") {
|
||||
t.Fatalf("workspace verification used an unsafe command: %s", call)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestVerifyRestoreWorkspaceRejectsInvalidOperatorResults(t *testing.T) {
|
||||
installation := preflightTestInstallation(t)
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
result compose.Result
|
||||
err error
|
||||
}{
|
||||
{name: "empty"},
|
||||
{name: "malformed", result: compose.Result{Stdout: `{malformed`}},
|
||||
{name: "trailing document", result: compose.Result{Stdout: `{"ready":true,"state":"active","workspaces":1}{}`}},
|
||||
{name: "unknown field", result: compose.Result{Stdout: `{"ready":true,"state":"active","workspaces":1,"detail":"unsafe"}`}},
|
||||
{name: "not ready", result: compose.Result{Stdout: `{"ready":false,"state":"uninitialized","workspaces":0}`}},
|
||||
{name: "unknown state", result: compose.Result{Stdout: `{"ready":true,"state":"unknown","workspaces":0}`}},
|
||||
{name: "inconsistent count", result: compose.Result{Stdout: `{"ready":true,"state":"uninitialized","workspaces":1}`}},
|
||||
{name: "nonzero", result: compose.Result{ExitCode: 2}, err: errors.New("exit status 2")},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
runner := &workspaceVerificationRunner{result: test.result, err: test.err}
|
||||
if err := verifyRestoreWorkspace(context.Background(), installation, runner); err == nil {
|
||||
t.Fatal("invalid workspace verification result was accepted")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func (lock fakeRestoreLock) Release() error {
|
||||
if lock.release != nil {
|
||||
lock.release()
|
||||
|
||||
Reference in New Issue
Block a user