feat(auth): integrate authentication with installation lifecycle

tools/tht/internal/backup/create.go

@@ -43,6 +43,15 @@ var requiredVolumes = []string{
 	"embedding-models",
 }
 
+var requiredServerVolumes = []string{"qdrant-data", "embedding-models"}
+
+func requiredBackupVolumes(installation config.Installation) []string {
+	if installation.Profile == "server" {
+		return requiredServerVolumes
+	}
+	return requiredVolumes
+}
+
 const (
 	helperImage       = "busybox:1.36.1"
 	drainPollInterval = time.Second
@@ -165,7 +174,7 @@ func createWithDependencies(ctx context.Context, installation config.Installatio
 	if err != nil {
 		return Result{}, err
 	}
-	volumes, err := inspectRequiredVolumes(ctx, dependencies.runner, rendered)
+	volumes, err := inspectRequiredVolumes(ctx, installation, dependencies.runner, rendered)
 	if err != nil {
 		return Result{}, err
 	}
@@ -408,7 +417,7 @@ func renderedConfiguration(ctx context.Context, installation config.Installation
 	if json.Unmarshal([]byte(result.Stdout), &rendered) != nil {
 		return renderedCompose{}, errors.New("Docker Compose returned invalid backup configuration")
 	}
-	for _, logical := range requiredVolumes {
+	for _, logical := range requiredBackupVolumes(installation) {
 		if rendered.Volumes[logical].Name == "" {
 			return renderedCompose{}, fmt.Errorf("required backup volume %q is not configured", logical)
 		}
@@ -416,9 +425,10 @@ func renderedConfiguration(ctx context.Context, installation config.Installation
 	return rendered, nil
 }
 
-func inspectRequiredVolumes(ctx context.Context, runner archiveRunner, rendered renderedCompose) ([]VolumeMetadata, error) {
-	names := make([]string, 0, len(requiredVolumes))
-	for _, logical := range requiredVolumes {
+func inspectRequiredVolumes(ctx context.Context, installation config.Installation, runner archiveRunner, rendered renderedCompose) ([]VolumeMetadata, error) {
+	required := requiredBackupVolumes(installation)
+	names := make([]string, 0, len(required))
+	for _, logical := range required {
 		names = append(names, rendered.Volumes[logical].Name)
 	}
 	result, err := runner.Run(ctx, append([]string{"volume", "inspect"}, names...), nil)
@@ -445,8 +455,8 @@ func inspectRequiredVolumes(ctx context.Context, runner archiveRunner, rendered
 			Labels map[string]string
 		}{item.Name, item.Driver, item.Labels}
 	}
-	volumes := make([]VolumeMetadata, 0, len(requiredVolumes))
-	for _, logical := range requiredVolumes {
+	volumes := make([]VolumeMetadata, 0, len(required))
+	for _, logical := range required {
 		name := rendered.Volumes[logical].Name
 		item, ok := byName[name]
 		if !ok || item.Driver == "" {
@@ -614,13 +624,12 @@ func normalizeBackupServiceStates(states []backupServiceState) ([]backupServiceS
 }
 
 func maintenance(ctx context.Context, installation config.Installation, runner archiveRunner, activate bool) error {
-	action := "deactivate"
+	action := "maintenance-deactivate"
 	if activate {
-		action = "activate"
+		action = "maintenance-activate"
 	}
 	result, err := runner.Run(ctx, installation.ComposeArgs(
-		"exec", "-T", "core", "curl", "-fsS", "--max-time", "5", "-X", "POST",
-		"http://127.0.0.1:8787/internal/maintenance/"+action,
+		"exec", "-T", "core", "node", "/app/backend/dist/operator-command.js", action,
 	), nil)
 	if err != nil {
 		return dockerError("change maintenance admissions", result, err)
@@ -631,8 +640,8 @@ func maintenance(ctx context.Context, installation config.Installation, runner a
 func waitForNoActiveSessions(ctx context.Context, installation config.Installation, runner archiveRunner, drain bool, sleep func(time.Duration)) error {
 	for attempt := 0; attempt < maxDrainPolls; attempt++ {
 		result, err := runner.Run(ctx, installation.ComposeArgs(
-			"exec", "-T", "core", "curl", "-fsS", "--max-time", "5",
-			"http://127.0.0.1:8787/sessions?scope="+runner.SessionInventoryScope(),
+			"exec", "-T", "core", "node", "/app/backend/dist/operator-command.js",
+			"session-inventory", runner.SessionInventoryScope(),
 		), nil)
 		if err != nil {
 			return dockerError("inspect active sessions", result, err)

tools/tht/internal/backup/create_test.go

@@ -501,6 +501,13 @@ func TestCreateIncludesServerPreservationRootsWithoutRecursingIntoBackupRoot(t *
 		t.Fatal(err)
 	}
 	archive := readFixtureArchive(t, output)
+	serverVolumes := make([]string, 0, len(archive.manifest.Volumes))
+	for _, volume := range archive.manifest.Volumes {
+		serverVolumes = append(serverVolumes, volume.LogicalName)
+	}
+	if strings.Join(serverVolumes, ",") != "embedding-models,qdrant-data" {
+		t.Fatalf("server backup volumes = %v, want only infrastructure named volumes", serverVolumes)
+	}
 	all := bytes.Join(mapValues(archive.files), nil)
 	for _, value := range []string{"session-state", "pi-state", "workspace-registry"} {
 		if !bytes.Contains(all, []byte(value)) {
@@ -659,7 +666,7 @@ func (runner *fakeBackupRunner) Run(_ context.Context, args []string, _ io.Reade
 	switch {
 	case strings.Contains(command, " config --format json"):
 		volumes := map[string]map[string]string{}
-		for _, logical := range requiredTestVolumes {
+		for _, logical := range requiredBackupVolumes(runner.installation) {
 			volumes[logical] = map[string]string{"name": runner.installation.ProjectName() + "_" + logical}
 		}
 		payload := map[string]any{
@@ -672,8 +679,9 @@ func (runner *fakeBackupRunner) Run(_ context.Context, args []string, _ io.Reade
 		encoded, _ := json.Marshal(payload)
 		return compose.Result{Stdout: string(encoded)}, nil
 	case strings.HasPrefix(command, "volume inspect "):
-		items := make([]map[string]any, 0, len(requiredTestVolumes))
-		for _, logical := range requiredTestVolumes {
+		required := requiredBackupVolumes(runner.installation)
+		items := make([]map[string]any, 0, len(required))
+		for _, logical := range required {
 			items = append(items, map[string]any{
 				"Name":   runner.installation.ProjectName() + "_" + logical,
 				"Driver": "local",
@@ -703,15 +711,15 @@ func (runner *fakeBackupRunner) Run(_ context.Context, args []string, _ io.Reade
 			lines = append(lines, fmt.Sprintf(`{"Service":%q,"State":%q}`, service, states[service]))
 		}
 		return compose.Result{Stdout: strings.Join(lines, "\n")}, nil
-	case strings.Contains(command, "/internal/maintenance/status"):
+	case strings.Contains(command, "operator-command.js maintenance-status"):
 		return compose.Result{Stdout: fmt.Sprintf(`{"active":%t,"admissions":0,"recoveryRequired":false}`, runner.maintenance)}, nil
-	case strings.Contains(command, "/internal/maintenance/activate"):
+	case strings.Contains(command, "operator-command.js maintenance-activate"):
 		runner.maintenance = true
 		return compose.Result{Stdout: `{"active":true,"admissions":0,"recoveryRequired":false}`}, nil
-	case strings.Contains(command, "/internal/maintenance/deactivate"):
+	case strings.Contains(command, "operator-command.js maintenance-deactivate"):
 		runner.maintenance = false
 		return compose.Result{Stdout: `{"active":false,"admissions":0,"recoveryRequired":false}`}, nil
-	case strings.Contains(command, "/sessions?scope="):
+	case strings.Contains(command, "operator-command.js session-inventory"):
 		if len(runner.sessionResponses) == 0 {
 			return compose.Result{Stdout: `[]`}, nil
 		}

tools/tht/internal/backup/restore.go

@@ -20,8 +20,10 @@ type RestoreRequest struct {
 	Drain   bool
 }
 
-// RestoreResult records the retained recovery point and the final service state.
+// RestoreResult records the final service state. Secret-bearing recovery checkpoints are always
+// destroyed internally and are therefore never exposed to callers.
 type RestoreResult struct {
+	// Deprecated: always empty. Recovery checkpoints are private transaction internals.
 	Checkpoint string
 	Restarted  bool
 	Verified   bool
@@ -33,6 +35,9 @@ type restoreVerify func(context.Context, config.Installation, archiveRunner) err
 type restoreDependencies struct {
 	preflight                func(context.Context, config.Installation, PreflightRequest) (PreflightResult, error)
 	checkpoint               func(context.Context, config.Installation, CreateRequest) (Result, error)
+	prepareRecovery          func(context.Context, config.Installation, string) (PreflightResult, error)
+	recover                  func(context.Context, config.Installation, PreflightResult, bool) error
+	cleanupCheckpoint        func(string) error
 	acquireLock              func(config.Installation) (restoreLock, error)
 	runner                   archiveRunner
 	sleep                    func(duration time.Duration)
@@ -56,7 +61,7 @@ func restoreWithDependencies(ctx context.Context, installation config.Installati
 	if request.Archive == "" {
 		return RestoreResult{}, errors.New("restore archive is required")
 	}
-	if deps.preflight == nil || deps.checkpoint == nil || deps.acquireLock == nil || deps.runner == nil || deps.restoreFile == nil || deps.restoreVolume == nil || deps.resetAuthenticationState == nil || deps.verify == nil {
+	if deps.preflight == nil || deps.checkpoint == nil || deps.prepareRecovery == nil || deps.recover == nil || deps.cleanupCheckpoint == nil || deps.acquireLock == nil || deps.runner == nil || deps.restoreFile == nil || deps.restoreVolume == nil || deps.resetAuthenticationState == nil || deps.verify == nil {
 		return RestoreResult{}, errors.New("restore dependencies are incomplete")
 	}
 	preflight, err := deps.preflight(ctx, installation, PreflightRequest{Archive: request.Archive, Confirm: true, AllowExternalSecrets: true})
@@ -69,11 +74,21 @@ func restoreWithDependencies(ctx context.Context, installation config.Installati
 		return RestoreResult{}, err
 	}
 
-	checkpoint, err := deps.checkpoint(ctx, installation, CreateRequest{})
+	checkpoint, err := deps.checkpoint(ctx, installation, CreateRequest{IncludeSecrets: true, Confirm: true})
 	if err != nil {
 		return RestoreResult{}, fmt.Errorf("create recovery checkpoint: %w", err)
 	}
-	result.Checkpoint = checkpoint.Path
+	recovery, err := deps.prepareRecovery(ctx, installation, checkpoint.Path)
+	if err != nil {
+		cleanupErr := deps.cleanupCheckpoint(checkpoint.Path)
+		return RestoreResult{}, errors.Join(fmt.Errorf("validate recovery checkpoint: %w", err), cleanupErr)
+	}
+	defer func() {
+		if cleanupErr := deps.cleanupCheckpoint(checkpoint.Path); cleanupErr != nil {
+			resultErr = errors.Join(resultErr, fmt.Errorf("destroy recovery checkpoint: %w", cleanupErr))
+		}
+	}()
+	defer recovery.CloseArchive()
 	lock, err := deps.acquireLock(installation)
 	if err != nil {
 		return result, err
@@ -91,7 +106,9 @@ func restoreWithDependencies(ctx context.Context, installation config.Installati
 	mutated := false
 	defer func() {
 		if resultErr != nil && mutated {
-			_ = runCompose(context.Background(), installation, deps.runner, "stop")
+			if recoveryErr := deps.recover(context.Background(), installation, recovery, wasRunning); recoveryErr != nil {
+				resultErr = errors.Join(resultErr, fmt.Errorf("restore recovery checkpoint: %w", recoveryErr))
+			}
 		}
 	}()
 	if wasRunning {
@@ -105,42 +122,9 @@ func restoreWithDependencies(ctx context.Context, installation config.Installati
 			return result, err
 		}
 	}
-	reader, err := zip.NewReader(archive, preflight.ArchiveSize)
-	if err != nil {
-		return result, fmt.Errorf("read verified restore archive: %w", err)
-	}
-	members := make(map[string]*zip.File, len(reader.File))
-	for _, member := range reader.File {
-		members[member.Name] = member
-	}
-	for _, entry := range preflight.Entries {
-		member := members[entry.Path]
-		if member == nil {
-			return result, fmt.Errorf("verified archive is missing %q", entry.Path)
-		}
-		stream, openErr := member.Open()
-		if openErr != nil {
-			return result, fmt.Errorf("open verified archive member %q: %w", entry.Path, openErr)
-		}
-		mutated = true
-		var restoreErr error
-		if entry.Kind == EntryVolume {
-			volume, found := restoreVolumeMetadata(preflight.Manifest, entry.LogicalName)
-			if !found {
-				_ = stream.Close()
-				return result, errors.New("verified volume metadata is incomplete")
-			}
-			restoreErr = deps.restoreVolume(ctx, installation, volume, stream)
-		} else {
-			restoreErr = deps.restoreFile(ctx, installation, entry, stream)
-		}
-		closeErr := stream.Close()
-		if restoreErr != nil {
-			return result, restoreErr
-		}
-		if closeErr != nil {
-			return result, closeErr
-		}
+	mutated = true
+	if err := restoreVerifiedEntries(ctx, installation, preflight, archive, deps.restoreFile, deps.restoreVolume); err != nil {
+		return result, err
 	}
 	if err := deps.resetAuthenticationState(ctx, installation, deps.runner); err != nil {
 		return result, fmt.Errorf("reset authentication state: %w", err)
@@ -164,6 +148,53 @@ func restoreWithDependencies(ctx context.Context, installation config.Installati
 	return result, nil
 }
 
+func restoreVerifiedEntries(
+	ctx context.Context,
+	installation config.Installation,
+	preflight PreflightResult,
+	archive io.ReaderAt,
+	restoreFile func(context.Context, config.Installation, ArchiveEntryMetadata, io.Reader) error,
+	restoreVolume func(context.Context, config.Installation, VolumeMetadata, io.Reader) error,
+) error {
+	reader, err := zip.NewReader(archive, preflight.ArchiveSize)
+	if err != nil {
+		return fmt.Errorf("read verified restore archive: %w", err)
+	}
+	members := make(map[string]*zip.File, len(reader.File))
+	for _, member := range reader.File {
+		members[member.Name] = member
+	}
+	for _, entry := range preflight.Entries {
+		member := members[entry.Path]
+		if member == nil {
+			return fmt.Errorf("verified archive is missing %q", entry.Path)
+		}
+		stream, openErr := member.Open()
+		if openErr != nil {
+			return fmt.Errorf("open verified archive member %q: %w", entry.Path, openErr)
+		}
+		var restoreErr error
+		if entry.Kind == EntryVolume {
+			volume, found := restoreVolumeMetadata(preflight.Manifest, entry.LogicalName)
+			if !found {
+				_ = stream.Close()
+				return errors.New("verified volume metadata is incomplete")
+			}
+			restoreErr = restoreVolume(ctx, installation, volume, stream)
+		} else {
+			restoreErr = restoreFile(ctx, installation, entry, stream)
+		}
+		closeErr := stream.Close()
+		if restoreErr != nil {
+			return restoreErr
+		}
+		if closeErr != nil {
+			return closeErr
+		}
+	}
+	return nil
+}
+
 func restoreVolumeMetadata(manifest Manifest, logicalName string) (VolumeMetadata, bool) {
 	if logicalName == "" {
 		return VolumeMetadata{}, false
@@ -177,12 +208,12 @@ func restoreVolumeMetadata(manifest Manifest, logicalName string) (VolumeMetadat
 }
 
 // resetAuthenticationState clears browser sessions and pending OIDC transactions without touching
-// installation-global auth.yaml or users.yaml. The command runs as the unprivileged core user so
-// the recreated state root is private to the service on both the local volume and server /data bind.
+// installation-global auth.yaml or users.yaml. A root-scoped one-shot repairs ownership and mode
+// before clearing children; links and malformed roots are rejected before any recursive removal.
 func resetAuthenticationState(ctx context.Context, installation config.Installation, runner archiveRunner) error {
 	result, err := runner.Run(ctx, installation.ComposeArgs(
-		"run", "--rm", "--no-deps", "--no-TTY", "--entrypoint", "sh", "core", "-ceu",
-		"find /data/auth -mindepth 1 -maxdepth 1 -exec rm -rf -- {} + && install -d -m 0700 /data/auth /data/auth/sessions /data/auth/oidc && test -z \"$(find /data/auth/sessions /data/auth/oidc -mindepth 1 -print -quit)\"",
+		"run", "--rm", "--no-deps", "--no-TTY", "--user", "0:0", "--entrypoint", "sh", "core", "-ceu",
+		"test ! -L /data/auth && { test ! -e /data/auth || test -d /data/auth; } && install -d -o 10001 -g 10001 -m 0700 /data/auth && find /data/auth -mindepth 1 -maxdepth 1 -exec rm -rf -- {} + && install -d -o 10001 -g 10001 -m 0700 /data/auth/sessions /data/auth/oidc && chmod 0700 /data/auth /data/auth/sessions /data/auth/oidc && chown 10001:10001 /data/auth /data/auth/sessions /data/auth/oidc && test -z \"$(find /data/auth/sessions /data/auth/oidc -mindepth 1 -print -quit)\"",
 	), nil)
 	if err != nil {
 		return dockerError("reset authentication state", result, err)

tools/tht/internal/backup/restore_host.go

@@ -5,7 +5,6 @@ import (
 	"crypto/rand"
 	"crypto/sha256"
 	"encoding/hex"
-	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
@@ -15,6 +14,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/aritmolab/thothii/tools/tht/internal/authconfig"
 	"github.com/aritmolab/thothii/tools/tht/internal/compose"
 	"github.com/aritmolab/thothii/tools/tht/internal/config"
 	"github.com/aritmolab/thothii/tools/tht/internal/doctor"
@@ -26,7 +26,7 @@ import (
 
 func productionRestoreDependencies(installation config.Installation) restoreDependencies {
 	runner := hostRunner{runner: compose.NewRunner(""), binary: "docker", profile: installation.Profile}
-	return restoreDependencies{
+	deps := restoreDependencies{
 		preflight: func(ctx context.Context, target config.Installation, request PreflightRequest) (PreflightResult, error) {
 			return Preflight(ctx, target, request, PreflightDependencies{
 				FreeBytes:                 restoreFreeBytes,
@@ -47,6 +47,7 @@ func productionRestoreDependencies(installation config.Installation) restoreDepe
 			request.Output = path
 			return Create(ctx, target, request)
 		},
+		cleanupCheckpoint: cleanupRecoveryCheckpoint,
 		acquireLock: func(target config.Installation) (restoreLock, error) {
 			return lifecycle.Acquire(target)
 		},
@@ -71,6 +72,46 @@ func productionRestoreDependencies(installation config.Installation) restoreDepe
 			"workspace": verifyRestoreWorkspace,
 		},
 	}
+	deps.prepareRecovery = func(ctx context.Context, target config.Installation, path string) (PreflightResult, error) {
+		return deps.preflight(ctx, target, PreflightRequest{Archive: path, Confirm: true, AllowExternalSecrets: true})
+	}
+	deps.recover = func(ctx context.Context, target config.Installation, recovery PreflightResult, wasRunning bool) error {
+		return recoverRestoreTransaction(ctx, target, recovery, wasRunning, deps)
+	}
+	return deps
+}
+
+func cleanupRecoveryCheckpoint(path string) error {
+	if err := safeio.RemoveCanonicalPrivateRegular(path); err != nil {
+		return errors.New("private recovery checkpoint could not be destroyed safely")
+	}
+	return nil
+}
+
+func recoverRestoreTransaction(ctx context.Context, installation config.Installation, recovery PreflightResult, wasRunning bool, deps restoreDependencies) error {
+	var resultErr error
+	if err := runCompose(ctx, installation, deps.runner, "stop"); err != nil {
+		resultErr = errors.Join(resultErr, err)
+	}
+	archive, err := recovery.RevalidateArchive()
+	if err != nil {
+		return errors.Join(resultErr, err)
+	}
+	if err := restoreVerifiedEntries(ctx, installation, recovery, archive, deps.restoreFile, deps.restoreVolume); err != nil {
+		return errors.Join(resultErr, err)
+	}
+	// Recovery restores only configuration/secret files and durable application volumes. Runtime
+	// authentication state is deliberately reset again so neither sessions nor OIDC transactions
+	// survive a failed restore attempt.
+	if err := deps.resetAuthenticationState(ctx, installation, deps.runner); err != nil {
+		return errors.Join(resultErr, err)
+	}
+	if wasRunning {
+		if err := composeStartAndVerify(ctx, installation, deps.runner); err != nil {
+			return errors.Join(resultErr, err)
+		}
+	}
+	return resultErr
 }
 
 func restoreCheckpointPath(installation config.Installation, now time.Time) (string, error) {
@@ -131,14 +172,14 @@ func validateRestoreReference(installation config.Installation, entry Entry) err
 }
 
 func validateRestoreVolumes(ctx context.Context, installation config.Installation, manifest Manifest, runner archiveRunner) error {
-	if len(manifest.Volumes) != len(requiredVolumes) {
+	if len(manifest.Volumes) != len(requiredBackupVolumes(installation)) {
 		return errors.New("backup volume set is incomplete")
 	}
 	rendered, err := renderedConfiguration(ctx, installation, runner)
 	if err != nil {
 		return err
 	}
-	current, err := inspectRequiredVolumes(ctx, runner, rendered)
+	current, err := inspectRequiredVolumes(ctx, installation, runner, rendered)
 	if err != nil {
 		return err
 	}
@@ -313,6 +354,13 @@ func verifyRestoreDoctor(ctx context.Context, installation config.Installation,
 		if configErr != nil {
 			return dockerError("verify restored Compose configuration", result, configErr)
 		}
+		diagnostics, diagnosticErr := authconfig.Check(ctx, installation, runner, false, false)
+		if diagnosticErr != nil || !diagnostics.Ready {
+			if diagnosticErr != nil {
+				return diagnosticErr
+			}
+			return errors.New("authentication diagnostics did not pass after restore")
+		}
 		return nil
 	}
 	report, err := doctor.Run(ctx, installation, runner)
@@ -320,7 +368,16 @@ func verifyRestoreDoctor(ctx context.Context, installation config.Installation,
 		return err
 	}
 	if !report.OK {
-		return errors.New("aggregate doctor did not pass after restore")
+		failed := make([]string, 0, len(report.Checks))
+		for _, check := range report.Checks {
+			if check.Status == doctor.StatusFailed {
+				failed = append(failed, check.Name)
+			}
+		}
+		if len(failed) == 0 {
+			return errors.New("aggregate doctor did not pass after restore")
+		}
+		return fmt.Errorf("aggregate doctor failed checks: %s", strings.Join(failed, ","))
 	}
 	return nil
 }
@@ -328,25 +385,36 @@ func verifyRestoreDoctor(ctx context.Context, installation config.Installation,
 func verifyRestorePi(ctx context.Context, installation config.Installation, runner archiveRunner) error {
 	running, err := restoreVerificationRunning(ctx, installation, runner)
 	if err != nil || !running {
-		return err
+		if err != nil {
+			return err
+		}
+		result, runErr := runner.Run(ctx, installation.ComposeArgs(
+			"run", "--rm", "--no-deps", "--no-TTY", "core", "pi", "--version",
+		), nil)
+		if runErr != nil || strings.TrimSpace(result.Stdout) == "" {
+			if runErr == nil {
+				runErr = errors.New("Pi version probe returned no version")
+			}
+			return dockerError("verify restored Pi runtime", result, runErr)
+		}
+		return nil
 	}
 	return pi.Doctor(ctx, compose.InstallationRunner{Installation: installation, Runner: runner})
 }
 
 func verifyRestoreWorkspace(ctx context.Context, installation config.Installation, runner archiveRunner) error {
 	running, err := restoreVerificationRunning(ctx, installation, runner)
-	if err != nil || !running {
+	if err != nil {
 		return err
 	}
-	result, err := runner.Run(ctx, installation.ComposeArgs(
-		"exec", "-T", "core", "curl", "-fsS", "--max-time", "5", "http://127.0.0.1:8787/workspaces",
-	), nil)
-	if err != nil {
-		return dockerError("inspect restored workspaces", result, err)
+	command := []string{"exec", "-T", "core", "node", "-e"}
+	if !running {
+		command = []string{"run", "--rm", "--no-deps", "--no-TTY", "core", "node", "-e"}
 	}
-	var workspaces []json.RawMessage
-	if json.Unmarshal([]byte(result.Stdout), &workspaces) != nil {
-		return errors.New("restored workspace inspection returned invalid JSON")
+	command = append(command, `const fs=require("node:fs");const p="/data/workspace-registry/state/active.json";const s=JSON.parse(fs.readFileSync(p,"utf8"));const h=/^[0-9a-f]{40}$/;if(!h.test(s.head)||!Array.isArray(s.revisions)||s.revisions.some(r=>!r||typeof r.id!=="string"||!r.id||!h.test(r.commit)||!h.test(r.blob)))process.exit(1);for(const r of s.revisions)fs.accessSync("/data/workspace-registry/snapshots/"+r.commit+"/"+r.id+".yaml",fs.constants.R_OK)`)
+	result, err := runner.Run(ctx, installation.ComposeArgs(command...), nil)
+	if err != nil {
+		return dockerError("validate restored workspace registry", result, err)
 	}
 	return nil
 }

tools/tht/internal/backup/restore_test.go

@@ -6,6 +6,7 @@ import (
 	"context"
 	"errors"
 	"io"
+	"os"
 	"path/filepath"
 	"strings"
 	"testing"
@@ -122,6 +123,14 @@ func TestRestoreStoppedInstallationRunsCheckpointRestoreAndVerification(t *testi
 		checkpointRequest = request
 		return Result{Path: "/tmp/checkpoint.zip"}, nil
 	}
+	deps.prepareRecovery = func(context.Context, config.Installation, string) (PreflightResult, error) {
+		events = append(events, "prepare-recovery")
+		return PreflightResult{}, nil
+	}
+	deps.cleanupCheckpoint = func(string) error {
+		events = append(events, "cleanup-checkpoint")
+		return nil
+	}
 	deps.acquireLock = func(config.Installation) (restoreLock, error) {
 		events = append(events, "lock")
 		return fakeRestoreLock{release: func() { events = append(events, "unlock") }}, nil
@@ -142,13 +151,13 @@ func TestRestoreStoppedInstallationRunsCheckpointRestoreAndVerification(t *testi
 	if err != nil {
 		t.Fatal(err)
 	}
-	if result.Checkpoint != "/tmp/checkpoint.zip" || result.Restarted || !result.Verified {
+	if result.Checkpoint != "" || result.Restarted || !result.Verified {
 		t.Fatalf("Restore() result = %#v", result)
 	}
-	if checkpointRequest.IncludeSecrets || checkpointRequest.Confirm {
-		t.Fatalf("checkpoint request = %#v, want non-secret unconfirmed checkpoint", checkpointRequest)
+	if !checkpointRequest.IncludeSecrets || !checkpointRequest.Confirm {
+		t.Fatalf("checkpoint request = %#v, want private confirmed secret-aware checkpoint", checkpointRequest)
 	}
-	if got, want := events, []string{"checkpoint", "lock", "file:configuration/operator.env", "health", "doctor", "pi", "workspace", "unlock"}; !equalStrings(got, want) {
+	if got, want := events, []string{"checkpoint", "prepare-recovery", "lock", "file:configuration/operator.env", "health", "doctor", "pi", "workspace", "unlock", "cleanup-checkpoint"}; !equalStrings(got, want) {
 		t.Fatalf("restore events = %v, want %v", got, want)
 	}
 }
@@ -196,9 +205,11 @@ func TestResetAuthenticationStateCreatesOnlyPrivateEmptyStateDirectories(t *test
 	}
 	joined := strings.Join(runner.args, "\x00")
 	for _, required := range []string{
-		"run", "--rm", "--no-deps", "--no-TTY", "--entrypoint", "sh", "core", "-ceu",
+		"run", "--rm", "--no-deps", "--no-TTY", "--user", "0:0", "--entrypoint", "sh", "core", "-ceu",
+		"test ! -L /data/auth",
+		"install -d -o 10001 -g 10001 -m 0700 /data/auth",
 		"find /data/auth -mindepth 1 -maxdepth 1 -exec rm -rf -- {} +",
-		"install -d -m 0700 /data/auth /data/auth/sessions /data/auth/oidc",
+		"install -d -o 10001 -g 10001 -m 0700 /data/auth/sessions /data/auth/oidc",
 		"find /data/auth/sessions /data/auth/oidc -mindepth 1 -print -quit",
 	} {
 		if !strings.Contains(joined, required) {
@@ -207,6 +218,16 @@ func TestResetAuthenticationStateCreatesOnlyPrivateEmptyStateDirectories(t *test
 	}
 }
 
+func TestResetAuthenticationStateFailsClosedForDamagedRoot(t *testing.T) {
+	installation := preflightTestInstallation(t)
+	runner := &authenticationStateResetRunner{result: compose.Result{ExitCode: 1}, err: errors.New("damaged auth root")}
+
+	err := resetAuthenticationState(context.Background(), installation, runner)
+	if err == nil || !strings.Contains(err.Error(), "reset authentication state") {
+		t.Fatalf("resetAuthenticationState() error = %v, want bounded fail-closed error", err)
+	}
+}
+
 func TestRestorePreflightFailureDoesNotMutateTarget(t *testing.T) {
 	installation := preflightTestInstallation(t)
 	runner := newBackupRunner(installation, true)
@@ -259,7 +280,7 @@ func TestRestoreCheckpointFailureDoesNotMutateTarget(t *testing.T) {
 	}
 }
 
-func TestRestoreFileFailureStopsMutatedTargetAndRetainsCheckpoint(t *testing.T) {
+func TestRestoreFileFailureRollsBackSecretAwareCheckpointBeforeCleanup(t *testing.T) {
 	installation := preflightTestInstallation(t)
 	archive := restoreArchive(t)
 	runner := newBackupRunner(installation, true)
@@ -268,7 +289,17 @@ func TestRestoreFileFailureStopsMutatedTargetAndRetainsCheckpoint(t *testing.T)
 	deps.checkpoint = func(context.Context, config.Installation, CreateRequest) (Result, error) {
 		return Result{Path: "/tmp/recovery.zip"}, nil
 	}
+	var events []string
+	deps.recover = func(context.Context, config.Installation, PreflightResult, bool) error {
+		events = append(events, "recover")
+		return nil
+	}
+	deps.cleanupCheckpoint = func(string) error {
+		events = append(events, "cleanup")
+		return nil
+	}
 	deps.restoreFile = func(context.Context, config.Installation, ArchiveEntryMetadata, io.Reader) error {
+		events = append(events, "mutate")
 		return fileErr
 	}
 
@@ -276,43 +307,145 @@ func TestRestoreFileFailureStopsMutatedTargetAndRetainsCheckpoint(t *testing.T)
 	if !errors.Is(err, fileErr) {
 		t.Fatalf("restore error = %v, want file error", err)
 	}
-	if result.Checkpoint != "/tmp/recovery.zip" {
-		t.Fatalf("recovery checkpoint = %q, want retained path", result.Checkpoint)
+	if result.Checkpoint != "" {
+		t.Fatalf("recovery checkpoint = %q, want no retained secret-bearing path", result.Checkpoint)
 	}
-	if runner.running || runner.stopCount != 2 {
-		t.Fatalf("mutated target was not stopped: running=%t stops=%d", runner.running, runner.stopCount)
+	if got, want := events, []string{"mutate", "recover", "cleanup"}; !equalStrings(got, want) {
+		t.Fatalf("failure recovery events = %v, want %v", got, want)
 	}
 }
 
-func TestRestoreStartFailureStopsRunningTarget(t *testing.T) {
+func TestRestoreFailureAfterAuthenticationMutationRollsBackAndClearsRuntimeState(t *testing.T) {
+	installation := preflightTestInstallation(t)
+	archive := restoreArchive(t)
+	runner := newBackupRunner(installation, false)
+	deps := restoreTestDependencies(t, runner)
+	resetErr := errors.New("authentication reset failed")
+	var events []string
+	deps.restoreFile = func(context.Context, config.Installation, ArchiveEntryMetadata, io.Reader) error {
+		events = append(events, "auth-config-mutated")
+		return nil
+	}
+	deps.resetAuthenticationState = func(context.Context, config.Installation, archiveRunner) error {
+		events = append(events, "auth-runtime-reset-failed")
+		return resetErr
+	}
+	deps.recover = func(context.Context, config.Installation, PreflightResult, bool) error {
+		events = append(events, "secret-aware-recovery-and-reauth-reset")
+		return nil
+	}
+	deps.cleanupCheckpoint = func(string) error {
+		events = append(events, "checkpoint-cleaned")
+		return nil
+	}
+
+	_, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: archive, Confirm: true}, deps)
+	if !errors.Is(err, resetErr) {
+		t.Fatalf("restore error = %v, want reset failure", err)
+	}
+	if got, want := events, []string{"auth-config-mutated", "auth-runtime-reset-failed", "secret-aware-recovery-and-reauth-reset", "checkpoint-cleaned"}; !equalStrings(got, want) {
+		t.Fatalf("post-auth recovery events = %v, want %v", got, want)
+	}
+}
+
+func TestRestoreCleanupFailureDoesNotSuppressRollback(t *testing.T) {
+	installation := preflightTestInstallation(t)
+	archive := restoreArchive(t)
+	deps := restoreTestDependencies(t, newBackupRunner(installation, false))
+	mutationErr := errors.New("post-mutation failure")
+	cleanupErr := errors.New("checkpoint cleanup failure")
+	recovered := false
+	deps.restoreFile = func(context.Context, config.Installation, ArchiveEntryMetadata, io.Reader) error { return mutationErr }
+	deps.recover = func(context.Context, config.Installation, PreflightResult, bool) error { recovered = true; return nil }
+	deps.cleanupCheckpoint = func(string) error { return cleanupErr }
+
+	_, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: archive, Confirm: true}, deps)
+	if !recovered || !errors.Is(err, mutationErr) || !errors.Is(err, cleanupErr) {
+		t.Fatalf("restore error = %v recovered=%t, want joined mutation/cleanup failure after rollback", err, recovered)
+	}
+}
+
+func TestRecoveryCheckpointCleanupRejectsSymlinksAndPermissiveFiles(t *testing.T) {
+	root := t.TempDir()
+	target := filepath.Join(root, "checkpoint.zip")
+	if err := os.WriteFile(target, []byte("secret"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	link := filepath.Join(root, "checkpoint-link.zip")
+	if err := os.Symlink(target, link); err != nil {
+		t.Skipf("symlink unavailable: %v", err)
+	}
+	if err := cleanupRecoveryCheckpoint(link); err == nil {
+		t.Fatal("cleanupRecoveryCheckpoint accepted a symlink")
+	}
+	if _, err := os.Stat(target); err != nil {
+		t.Fatalf("symlink target was changed: %v", err)
+	}
+	if err := os.Chmod(target, 0o644); err != nil {
+		t.Fatal(err)
+	}
+	if err := cleanupRecoveryCheckpoint(target); err == nil {
+		t.Fatal("cleanupRecoveryCheckpoint accepted a permissive secret checkpoint")
+	}
+}
+
+func TestRecoveryCheckpointCleanupRemovesOnlyPrivateRegularFile(t *testing.T) {
+	root, err := filepath.EvalSymlinks(t.TempDir())
+	if err != nil {
+		t.Fatal(err)
+	}
+	checkpoint := filepath.Join(root, "checkpoint.zip")
+	if err := os.WriteFile(checkpoint, []byte("secret checkpoint"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	if err := cleanupRecoveryCheckpoint(checkpoint); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := os.Lstat(checkpoint); !errors.Is(err, os.ErrNotExist) {
+		t.Fatalf("private checkpoint still exists after cleanup: %v", err)
+	}
+}
+
+func TestRestoreStartFailureRecoversPreviouslyRunningTarget(t *testing.T) {
 	installation := preflightTestInstallation(t)
 	archive := restoreArchive(t)
 	backingRunner := newBackupRunner(installation, true)
 	deps := restoreTestDependencies(t, failStartRestoreRunner{fakeBackupRunner: backingRunner})
+	recovered := false
+	deps.recover = func(context.Context, config.Installation, PreflightResult, bool) error {
+		recovered = true
+		backingRunner.running = true
+		return nil
+	}
 
 	_, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: archive, Confirm: true}, deps)
 	if err == nil || !strings.Contains(err.Error(), "start refused") {
 		t.Fatalf("restore error = %v, want restart failure", err)
 	}
-	if backingRunner.running || backingRunner.stopCount != 2 || backingRunner.startCount != 0 {
-		t.Fatalf("failed restart left target available: running=%t stops=%d starts=%d", backingRunner.running, backingRunner.stopCount, backingRunner.startCount)
+	if !recovered || !backingRunner.running {
+		t.Fatalf("failed restart was not rolled back: recovered=%t running=%t", recovered, backingRunner.running)
 	}
 }
 
-func TestRestoreVerificationFailureStopsRunningTarget(t *testing.T) {
+func TestRestoreVerificationFailureRecoversPreviouslyRunningTarget(t *testing.T) {
 	installation := preflightTestInstallation(t)
 	archive := restoreArchive(t)
 	runner := newBackupRunner(installation, true)
 	deps := restoreTestDependencies(t, runner)
 	verificationErr := errors.New("Pi is unavailable")
 	deps.verify["pi"] = func(context.Context, config.Installation, archiveRunner) error { return verificationErr }
+	recovered := false
+	deps.recover = func(context.Context, config.Installation, PreflightResult, bool) error {
+		recovered = true
+		return nil
+	}
 
 	_, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: archive, Confirm: true}, deps)
 	if !errors.Is(err, verificationErr) {
 		t.Fatalf("restore error = %v, want verification failure", err)
 	}
-	if runner.running || runner.stopCount != 2 || runner.startCount != 1 {
-		t.Fatalf("verification failure left target available: running=%t stops=%d starts=%d", runner.running, runner.stopCount, runner.startCount)
+	if !recovered || !runner.running {
+		t.Fatalf("verification failure was not rolled back: recovered=%t running=%t", recovered, runner.running)
 	}
 }
 
@@ -361,11 +494,15 @@ type fakeRestoreLock struct {
 	release func()
 }
 
-type authenticationStateResetRunner struct{ args []string }
+type authenticationStateResetRunner struct {
+	args   []string
+	result compose.Result
+	err    error
+}
 
 func (runner *authenticationStateResetRunner) Run(_ context.Context, args []string, _ io.Reader) (compose.Result, error) {
 	runner.args = append([]string(nil), args...)
-	return compose.Result{}, nil
+	return runner.result, runner.err
 }
 
 func (runner *authenticationStateResetRunner) Stream(context.Context, []string, io.Reader, io.Writer) (compose.Result, error) {
@@ -402,10 +539,15 @@ func restoreTestDependencies(t *testing.T, runner archiveRunner) restoreDependen
 		checkpoint: func(context.Context, config.Installation, CreateRequest) (Result, error) {
 			return Result{Path: "/tmp/default-checkpoint.zip"}, nil
 		},
-		acquireLock: func(config.Installation) (restoreLock, error) { return fakeRestoreLock{}, nil },
-		runner:      runner,
-		sleep:       func(time.Duration) {},
-		restoreFile: func(context.Context, config.Installation, ArchiveEntryMetadata, io.Reader) error { return nil },
+		prepareRecovery: func(context.Context, config.Installation, string) (PreflightResult, error) {
+			return PreflightResult{}, nil
+		},
+		recover:           func(context.Context, config.Installation, PreflightResult, bool) error { return nil },
+		cleanupCheckpoint: func(string) error { return nil },
+		acquireLock:       func(config.Installation) (restoreLock, error) { return fakeRestoreLock{}, nil },
+		runner:            runner,
+		sleep:             func(time.Duration) {},
+		restoreFile:       func(context.Context, config.Installation, ArchiveEntryMetadata, io.Reader) error { return nil },
 		restoreVolume: func(context.Context, config.Installation, VolumeMetadata, io.Reader) error {
 			return nil
 		},