ThothII/tools/tht/internal/backup/restore_test.go

package backup

import (
	"archive/tar"
	"bytes"
	"context"
	"errors"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strings"
	"testing"
	"time"

	"github.com/aritmolab/thothii/tools/tht/internal/compose"
	"github.com/aritmolab/thothii/tools/tht/internal/config"
)

func TestRestorePublicPathUsesConcreteProductionPreflight(t *testing.T) {
	root := t.TempDir()
	installation := config.Installation{
		Path:             filepath.Join(root, "deploy", "local-dev", "thothii-installation.yaml"),
		ProjectDirectory: root,
	}
	missing := filepath.Join(root, "missing.zip")

	_, err := Restore(context.Background(), installation, RestoreRequest{Archive: missing, Confirm: true})

	if err == nil || strings.Contains(err.Error(), "dependencies are unavailable") || !strings.Contains(err.Error(), "backup archive") {
		t.Fatalf("Restore() error = %v, want production archive preflight", err)
	}
}

func TestVolumeRestoreCommandKeepsTarInputOpenWithoutTTY(t *testing.T) {
	args := volumeRestoreCommand("project_sessions")
	wantPrefix := []string{"run", "--rm", "--interactive", "--network", "none"}
	if len(args) < len(wantPrefix) || !equalStrings(args[:len(wantPrefix)], wantPrefix) {
		t.Fatalf("volumeRestoreCommand() prefix = %q, want %q", args, wantPrefix)
	}
	for _, arg := range args {
		if arg == "--tty" || arg == "-t" {
			t.Fatalf("volumeRestoreCommand() requests a TTY: %q", args)
		}
	}
}

func TestRestoreRestoresVerifiedVolumesInManifestOrderBeforeAuthenticationReset(t *testing.T) {
	installation := preflightTestInstallation(t)
	archive := filepath.Join(t.TempDir(), "restore-volumes.zip")
	sessionsTar := safeRestoreTar(t, "session.txt", "session")
	settingsTar := safeRestoreTar(t, "settings.json", "settings")
	writePreflightArchive(t, archive, preflightArchiveSpec{
		volumes: []VolumeMetadata{
			{LogicalName: "sessions", Name: "project_sessions", Driver: "local"},
			{LogicalName: "settings", Name: "project_settings", Driver: "local"},
		},
		entries: []preflightArchiveEntry{
			{path: "configuration/operator.env", body: []byte("safe")},
			{path: "volumes/settings.tar", body: settingsTar, kind: EntryVolume, owner: "volume:settings", logicalName: "settings"},
			{path: "volumes/sessions.tar", body: sessionsTar, kind: EntryVolume, owner: "volume:sessions", logicalName: "sessions"},
		},
	})
	runner := newBackupRunner(installation, false)
	deps := restoreTestDependencies(t, runner)
	var events []string
	deps.restoreFile = func(_ context.Context, _ config.Installation, entry ArchiveEntryMetadata, _ io.Reader) error {
		events = append(events, "file:"+entry.Path)
		return nil
	}
	deps.restoreVolume = func(_ context.Context, _ config.Installation, volume VolumeMetadata, stream io.Reader) error {
		if _, err := io.ReadAll(stream); err != nil {
			return err
		}
		events = append(events, "volume:"+volume.LogicalName)
		return nil
	}
	deps.resetAuthenticationState = func(context.Context, config.Installation, archiveRunner) error {
		events = append(events, "reset-auth-state")
		return nil
	}

	if _, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: archive, Confirm: true}, deps); err != nil {
		t.Fatal(err)
	}
	if got, want := events, []string{
		"file:configuration/operator.env",
		"volume:sessions",
		"volume:settings",
		"reset-auth-state",
	}; !equalStrings(got, want) {
		t.Fatalf("restore events = %v, want %v", got, want)
	}
}

func safeRestoreTar(t *testing.T, name, contents string) []byte {
	t.Helper()
	var output bytes.Buffer
	writer := tar.NewWriter(&output)
	if err := writer.WriteHeader(&tar.Header{Name: name, Mode: 0o600, Size: int64(len(contents)), Typeflag: tar.TypeReg}); err != nil {
		t.Fatal(err)
	}
	if _, err := writer.Write([]byte(contents)); err != nil {
		t.Fatal(err)
	}
	if err := writer.Close(); err != nil {
		t.Fatal(err)
	}
	return output.Bytes()
}

func TestRestoreStoppedInstallationRunsCheckpointRestoreAndVerification(t *testing.T) {
	installation := preflightTestInstallation(t)
	archive := filepath.Join(t.TempDir(), "restore.zip")
	writePreflightArchive(t, archive, preflightArchiveSpec{
		entries: []preflightArchiveEntry{{path: "configuration/operator.env", body: []byte("safe")}},
	})
	runner := newBackupRunner(installation, false)
	var events []string
	var checkpointRequest CreateRequest
	deps := restoreTestDependencies(t, runner)
	deps.checkpoint = func(_ context.Context, _ config.Installation, request CreateRequest) (Result, error) {
		events = append(events, "checkpoint")
		checkpointRequest = request
		return Result{Path: "/tmp/checkpoint.zip"}, nil
	}
	deps.prepareRecovery = func(context.Context, config.Installation, string) (PreflightResult, error) {
		events = append(events, "prepare-recovery")
		return PreflightResult{}, nil
	}
	deps.cleanupCheckpoint = func(string) error {
		events = append(events, "cleanup-checkpoint")
		return nil
	}
	deps.acquireLock = func(config.Installation) (restoreLock, error) {
		events = append(events, "lock")
		return fakeRestoreLock{release: func() { events = append(events, "unlock") }}, nil
	}
	deps.restoreFile = func(_ context.Context, _ config.Installation, entry ArchiveEntryMetadata, _ io.Reader) error {
		events = append(events, "file:"+entry.Path)
		return nil
	}
	for _, name := range []string{"health", "doctor", "pi", "workspace"} {
		name := name
		deps.verify[name] = func(context.Context, config.Installation, archiveRunner) error {
			events = append(events, name)
			return nil
		}
	}

	result, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: archive, Confirm: true}, deps)
	if err != nil {
		t.Fatal(err)
	}
	if result.Checkpoint != "" || result.Restarted || !result.Verified {
		t.Fatalf("Restore() result = %#v", result)
	}
	if !checkpointRequest.IncludeSecrets || !checkpointRequest.Confirm {
		t.Fatalf("checkpoint request = %#v, want private confirmed secret-aware checkpoint", checkpointRequest)
	}
	if got, want := events, []string{"checkpoint", "prepare-recovery", "lock", "file:configuration/operator.env", "health", "doctor", "pi", "workspace", "unlock", "cleanup-checkpoint"}; !equalStrings(got, want) {
		t.Fatalf("restore events = %v, want %v", got, want)
	}
}

func TestRestoreResetsAuthenticationStateBeforeRestart(t *testing.T) {
	installation := preflightTestInstallation(t)
	archive := restoreArchive(t)
	runner := newBackupRunner(installation, true)
	deps := restoreTestDependencies(t, runner)
	var events []string
	deps.restoreFile = func(_ context.Context, _ config.Installation, entry ArchiveEntryMetadata, _ io.Reader) error {
		events = append(events, "file:"+entry.Path)
		return nil
	}
	deps.resetAuthenticationState = func(context.Context, config.Installation, archiveRunner) error {
		events = append(events, "reset-auth-state")
		return nil
	}
	for _, name := range []string{"health", "doctor", "pi", "workspace"} {
		name := name
		deps.verify[name] = func(context.Context, config.Installation, archiveRunner) error {
			events = append(events, name)
			return nil
		}
	}

	result, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: archive, Confirm: true}, deps)
	if err != nil {
		t.Fatal(err)
	}
	if !result.Restarted || !result.Verified {
		t.Fatalf("restore result = %#v", result)
	}
	if got, want := events, []string{"file:configuration/operator.env", "reset-auth-state", "health", "doctor", "pi", "workspace"}; !equalStrings(got, want) {
		t.Fatalf("restore events = %v, want %v", got, want)
	}
}

func TestResetAuthenticationStateCreatesOnlyPrivateEmptyStateDirectories(t *testing.T) {
	installation := preflightTestInstallation(t)
	runner := &authenticationStateResetRunner{}

	if err := resetAuthenticationState(context.Background(), installation, runner); err != nil {
		t.Fatal(err)
	}
	joined := strings.Join(runner.args, "\x00")
	for _, required := range []string{
		"run", "--rm", "--no-deps", "--no-TTY", "--user", "0:0", "--entrypoint", "sh", "core", "-ceu",
		"test ! -L /data/auth",
		"install -d -o 10001 -g 10001 -m 0700 /data/auth",
		"find /data/auth -mindepth 1 -maxdepth 1 -exec rm -rf -- {} +",
		"install -d -o 10001 -g 10001 -m 0700 /data/auth/sessions /data/auth/oidc",
		"find /data/auth/sessions /data/auth/oidc -mindepth 1 -print -quit",
	} {
		if !strings.Contains(joined, required) {
			t.Fatalf("authentication state reset command omits %q: %#v", required, runner.args)
		}
	}
}

func TestResetAuthenticationStateFailsClosedForDamagedRoot(t *testing.T) {
	installation := preflightTestInstallation(t)
	runner := &authenticationStateResetRunner{result: compose.Result{ExitCode: 1}, err: errors.New("damaged auth root")}

	err := resetAuthenticationState(context.Background(), installation, runner)
	if err == nil || !strings.Contains(err.Error(), "reset authentication state") {
		t.Fatalf("resetAuthenticationState() error = %v, want bounded fail-closed error", err)
	}
}

func TestRestorePreflightFailureDoesNotMutateTarget(t *testing.T) {
	installation := preflightTestInstallation(t)
	runner := newBackupRunner(installation, true)
	deps := restoreTestDependencies(t, runner)
	preflightErr := errors.New("archive cannot be restored")
	deps.preflight = func(context.Context, config.Installation, PreflightRequest) (PreflightResult, error) {
		return PreflightResult{}, preflightErr
	}
	checkpointCalls := 0
	deps.checkpoint = func(context.Context, config.Installation, CreateRequest) (Result, error) {
		checkpointCalls++
		return Result{}, nil
	}
	restoredFiles := 0
	deps.restoreFile = func(context.Context, config.Installation, ArchiveEntryMetadata, io.Reader) error {
		restoredFiles++
		return nil
	}

	result, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: "unreadable.zip", Confirm: true}, deps)
	if !errors.Is(err, preflightErr) {
		t.Fatalf("restore error = %v, want preflight error", err)
	}
	if result != (RestoreResult{}) || checkpointCalls != 0 || restoredFiles != 0 || runner.stopCount != 0 || runner.startCount != 0 || !runner.running {
		t.Fatalf("preflight failure mutated target: result=%#v checkpoint=%d files=%d stops=%d starts=%d running=%t", result, checkpointCalls, restoredFiles, runner.stopCount, runner.startCount, runner.running)
	}
}

func TestRestoreCheckpointFailureDoesNotMutateTarget(t *testing.T) {
	installation := preflightTestInstallation(t)
	archive := restoreArchive(t)
	runner := newBackupRunner(installation, true)
	deps := restoreTestDependencies(t, runner)
	checkpointErr := errors.New("checkpoint unavailable")
	deps.checkpoint = func(context.Context, config.Installation, CreateRequest) (Result, error) {
		return Result{}, checkpointErr
	}
	restoredFiles := 0
	deps.restoreFile = func(context.Context, config.Installation, ArchiveEntryMetadata, io.Reader) error {
		restoredFiles++
		return nil
	}

	result, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: archive, Confirm: true}, deps)
	if !errors.Is(err, checkpointErr) {
		t.Fatalf("restore error = %v, want checkpoint error", err)
	}
	if result != (RestoreResult{}) || restoredFiles != 0 || runner.stopCount != 0 || runner.startCount != 0 || !runner.running {
		t.Fatalf("checkpoint failure mutated target: result=%#v files=%d stops=%d starts=%d running=%t", result, restoredFiles, runner.stopCount, runner.startCount, runner.running)
	}
}

func TestRestoreFileFailureRollsBackSecretAwareCheckpointBeforeCleanup(t *testing.T) {
	installation := preflightTestInstallation(t)
	archive := restoreArchive(t)
	runner := newBackupRunner(installation, true)
	deps := restoreTestDependencies(t, runner)
	fileErr := errors.New("cannot restore operator configuration")
	deps.checkpoint = func(context.Context, config.Installation, CreateRequest) (Result, error) {
		return Result{Path: "/tmp/recovery.zip"}, nil
	}
	var events []string
	deps.recover = func(context.Context, config.Installation, PreflightResult, bool) error {
		events = append(events, "recover")
		return nil
	}
	deps.cleanupCheckpoint = func(string) error {
		events = append(events, "cleanup")
		return nil
	}
	deps.restoreFile = func(context.Context, config.Installation, ArchiveEntryMetadata, io.Reader) error {
		events = append(events, "mutate")
		return fileErr
	}

	result, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: archive, Confirm: true}, deps)
	if !errors.Is(err, fileErr) {
		t.Fatalf("restore error = %v, want file error", err)
	}
	if result.Checkpoint != "" {
		t.Fatalf("recovery checkpoint = %q, want no retained secret-bearing path", result.Checkpoint)
	}
	if got, want := events, []string{"mutate", "recover", "cleanup"}; !equalStrings(got, want) {
		t.Fatalf("failure recovery events = %v, want %v", got, want)
	}
}

func TestRestoreFailureAfterAuthenticationMutationRollsBackAndClearsRuntimeState(t *testing.T) {
	installation := preflightTestInstallation(t)
	archive := restoreArchive(t)
	runner := newBackupRunner(installation, false)
	deps := restoreTestDependencies(t, runner)
	resetErr := errors.New("authentication reset failed")
	var events []string
	deps.restoreFile = func(context.Context, config.Installation, ArchiveEntryMetadata, io.Reader) error {
		events = append(events, "auth-config-mutated")
		return nil
	}
	deps.resetAuthenticationState = func(context.Context, config.Installation, archiveRunner) error {
		events = append(events, "auth-runtime-reset-failed")
		return resetErr
	}
	deps.recover = func(context.Context, config.Installation, PreflightResult, bool) error {
		events = append(events, "secret-aware-recovery-and-reauth-reset")
		return nil
	}
	deps.cleanupCheckpoint = func(string) error {
		events = append(events, "checkpoint-cleaned")
		return nil
	}

	_, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: archive, Confirm: true}, deps)
	if !errors.Is(err, resetErr) {
		t.Fatalf("restore error = %v, want reset failure", err)
	}
	if got, want := events, []string{"auth-config-mutated", "auth-runtime-reset-failed", "secret-aware-recovery-and-reauth-reset", "checkpoint-cleaned"}; !equalStrings(got, want) {
		t.Fatalf("post-auth recovery events = %v, want %v", got, want)
	}
}

func TestRestoreCleanupFailureDoesNotSuppressRollback(t *testing.T) {
	installation := preflightTestInstallation(t)
	archive := restoreArchive(t)
	deps := restoreTestDependencies(t, newBackupRunner(installation, false))
	mutationErr := errors.New("post-mutation failure")
	cleanupErr := errors.New("checkpoint cleanup failure")
	recovered := false
	deps.restoreFile = func(context.Context, config.Installation, ArchiveEntryMetadata, io.Reader) error { return mutationErr }
	deps.recover = func(context.Context, config.Installation, PreflightResult, bool) error { recovered = true; return nil }
	deps.cleanupCheckpoint = func(string) error { return cleanupErr }

	_, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: archive, Confirm: true}, deps)
	if !recovered || !errors.Is(err, mutationErr) || !errors.Is(err, cleanupErr) {
		t.Fatalf("restore error = %v recovered=%t, want joined mutation/cleanup failure after rollback", err, recovered)
	}
}

func TestRecoveryCheckpointCleanupRejectsSymlinksAndPermissiveFiles(t *testing.T) {
	root := t.TempDir()
	target := filepath.Join(root, "checkpoint.zip")
	if err := os.WriteFile(target, []byte("secret"), 0o600); err != nil {
		t.Fatal(err)
	}
	link := filepath.Join(root, "checkpoint-link.zip")
	if err := os.Symlink(target, link); err != nil {
		t.Skipf("symlink unavailable: %v", err)
	}
	if err := cleanupRecoveryCheckpoint(link); err == nil {
		t.Fatal("cleanupRecoveryCheckpoint accepted a symlink")
	}
	if _, err := os.Stat(target); err != nil {
		t.Fatalf("symlink target was changed: %v", err)
	}
	if err := os.Chmod(target, 0o644); err != nil {
		t.Fatal(err)
	}
	if err := cleanupRecoveryCheckpoint(target); err == nil {
		t.Fatal("cleanupRecoveryCheckpoint accepted a permissive secret checkpoint")
	}
}

func TestRecoveryCheckpointCleanupRemovesOnlyPrivateRegularFile(t *testing.T) {
	root, err := filepath.EvalSymlinks(t.TempDir())
	if err != nil {
		t.Fatal(err)
	}
	checkpoint := filepath.Join(root, "checkpoint.zip")
	if err := os.WriteFile(checkpoint, []byte("secret checkpoint"), 0o600); err != nil {
		t.Fatal(err)
	}
	if err := cleanupRecoveryCheckpoint(checkpoint); err != nil {
		t.Fatal(err)
	}
	if _, err := os.Lstat(checkpoint); !errors.Is(err, os.ErrNotExist) {
		t.Fatalf("private checkpoint still exists after cleanup: %v", err)
	}
}

func TestRestoreStartFailureRecoversPreviouslyRunningTarget(t *testing.T) {
	installation := preflightTestInstallation(t)
	archive := restoreArchive(t)
	backingRunner := newBackupRunner(installation, true)
	deps := restoreTestDependencies(t, failStartRestoreRunner{fakeBackupRunner: backingRunner})
	recovered := false
	deps.recover = func(context.Context, config.Installation, PreflightResult, bool) error {
		recovered = true
		backingRunner.running = true
		return nil
	}

	_, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: archive, Confirm: true}, deps)
	if err == nil || !strings.Contains(err.Error(), "start refused") {
		t.Fatalf("restore error = %v, want restart failure", err)
	}
	if !recovered || !backingRunner.running {
		t.Fatalf("failed restart was not rolled back: recovered=%t running=%t", recovered, backingRunner.running)
	}
}

func TestRestoreVerificationFailureRecoversPreviouslyRunningTarget(t *testing.T) {
	installation := preflightTestInstallation(t)
	archive := restoreArchive(t)
	runner := newBackupRunner(installation, true)
	deps := restoreTestDependencies(t, runner)
	verificationErr := errors.New("Pi is unavailable")
	deps.verify["pi"] = func(context.Context, config.Installation, archiveRunner) error { return verificationErr }
	recovered := false
	deps.recover = func(context.Context, config.Installation, PreflightResult, bool) error {
		recovered = true
		return nil
	}

	_, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: archive, Confirm: true}, deps)
	if !errors.Is(err, verificationErr) {
		t.Fatalf("restore error = %v, want verification failure", err)
	}
	if !recovered || !runner.running {
		t.Fatalf("verification failure was not rolled back: recovered=%t running=%t", recovered, runner.running)
	}
}

func TestRestoreKeepsAdmissionBarrierActiveUntilVerificationCommits(t *testing.T) {
	installation := preflightTestInstallation(t)
	archive := restoreArchive(t)
	runner := newBackupRunner(installation, true)
	deps := restoreTestDependencies(t, runner)
	verificationEntered := make(chan struct{})
	allowVerification := make(chan struct{})
	deps.verify["health"] = func(context.Context, config.Installation, archiveRunner) error {
		close(verificationEntered)
		<-allowVerification
		if !runner.maintenance {
			return errors.New("maintenance barrier was removed before verification committed")
		}
		return nil
	}

	type restoreOutcome struct {
		result RestoreResult
		err    error
	}
	done := make(chan restoreOutcome, 1)
	go func() {
		result, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: archive, Confirm: true}, deps)
		done <- restoreOutcome{result: result, err: err}
	}()
	select {
	case <-verificationEntered:
	case <-time.After(2 * time.Second):
		t.Fatal("restore did not reach post-start verification")
	}

	// A newly admitted operation observes the same durable barrier as the backend gate. No
	// operation may enter after restore mutation and before the verification transaction commits.
	admissionAttempt := make(chan bool, 1)
	go func() {
		admissionAttempt <- !runner.maintenance
	}()
	admissionAllowed := <-admissionAttempt
	close(allowVerification)
	outcome := <-done
	if admissionAllowed {
		t.Fatal("a new operation could enter while restore verification was still in progress")
	}
	if outcome.err != nil || !outcome.result.Verified {
		t.Fatalf("restore result = %#v, %v; want successful verified restore", outcome.result, outcome.err)
	}
	if runner.maintenance {
		t.Fatal("maintenance barrier remained active after successful verification")
	}
}

func TestRestoreRecoversBehindBarrierForEveryVerificationFailure(t *testing.T) {
	for _, verification := range []string{"health", "doctor", "pi", "workspace"} {
		verification := verification
		t.Run(verification, func(t *testing.T) {
			installation := preflightTestInstallation(t)
			runner := newBackupRunner(installation, true)
			deps := restoreTestDependencies(t, runner)
			verificationErr := fmt.Errorf("%s verification failed", verification)
			deps.verify[verification] = func(context.Context, config.Installation, archiveRunner) error {
				return verificationErr
			}
			var recoveryBarrierActive bool
			var recoveryContext cleanupContextObservation
			deps.recover = func(ctx context.Context, _ config.Installation, _ PreflightResult, _ bool) error {
				recoveryContext = observeCleanupContext(ctx)
				recoveryBarrierActive = runner.maintenance
				runner.running, runner.coreRunning = true, true
				return nil
			}

			_, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: restoreArchive(t), Confirm: true}, deps)
			if !errors.Is(err, verificationErr) {
				t.Fatalf("Restore() error = %v, want %v", err, verificationErr)
			}
			if !recoveryBarrierActive {
				t.Fatal("recovery started after the admission barrier had been removed")
			}
			assertIndependentBoundedCleanupContexts(t, []cleanupContextObservation{recoveryContext})
			if runner.maintenance || !runner.running {
				t.Fatalf("recovery cleanup state = maintenance:%t running:%t", runner.maintenance, runner.running)
			}
		})
	}
}

func TestRestoreDoesNotRollbackAfterFinalDeactivationResponseLoss(t *testing.T) {
	installation := preflightTestInstallation(t)
	backing := newBackupRunner(installation, true)
	deactivationResponseLost := errors.New("deactivation response lost")
	runner := &commandFailureRunner{
		fakeBackupRunner: backing,
		failures: []*commandFailure{{
			match: func(command string) bool { return strings.Contains(command, " maintenance-deactivate") },
			err:   deactivationResponseLost,
			effect: func() {
				// The barrier was removed, but Docker lost the response to the operator command.
				backing.maintenance = false
			},
			remaining: 1,
		}},
	}
	deps := restoreTestDependencies(t, runner)
	recoveryCalls := 0
	deps.recover = func(context.Context, config.Installation, PreflightResult, bool) error {
		recoveryCalls++
		return nil
	}

	_, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: restoreArchive(t), Confirm: true}, deps)
	if !errors.Is(err, deactivationResponseLost) {
		t.Fatalf("Restore() error = %v, want %v", err, deactivationResponseLost)
	}
	if recoveryCalls != 0 {
		t.Fatalf("successful verification triggered an unnecessary rollback: %d recoveries", recoveryCalls)
	}
	if backing.maintenance || !backing.running {
		t.Fatalf("final deactivation cleanup state = maintenance:%t running:%t", backing.maintenance, backing.running)
	}
}

func TestRestoreUsesBoundedIndependentCleanupContextsAfterCanceledStopResponse(t *testing.T) {
	installation := preflightTestInstallation(t)
	backing := newBackupRunner(installation, true)
	caller, cancel := context.WithCancel(context.Background())
	t.Cleanup(cancel)
	runner := &commandFailureRunner{
		fakeBackupRunner: backing,
		failures: []*commandFailure{{
			match: func(command string) bool { return strings.HasSuffix(command, " stop") },
			err:   context.Canceled,
			effect: func() {
				backing.stopCount++
				backing.running, backing.coreRunning = false, false
				cancel()
			},
			remaining: 1,
		}},
	}

	_, err := restoreWithDependencies(caller, installation, RestoreRequest{Archive: restoreArchive(t), Confirm: true}, restoreTestDependencies(t, runner))
	if !errors.Is(err, context.Canceled) {
		t.Fatalf("Restore() error = %v, want context cancellation", err)
	}
	if backing.maintenance || !backing.running || backing.startCount != 1 {
		t.Fatalf("cancelled cleanup state = maintenance:%t running:%t starts:%d", backing.maintenance, backing.running, backing.startCount)
	}
	assertIndependentBoundedCleanupContexts(t, runner.cleanupCommandContexts)
}

func TestRestoreUsesBoundedRecoveryContextAfterPostMutationCancellation(t *testing.T) {
	installation := preflightTestInstallation(t)
	runner := newBackupRunner(installation, true)
	caller, cancel := context.WithCancel(context.Background())
	t.Cleanup(cancel)
	deps := restoreTestDependencies(t, runner)
	deps.restoreFile = func(context.Context, config.Installation, ArchiveEntryMetadata, io.Reader) error {
		cancel()
		return context.Canceled
	}
	var recoveryContext cleanupContextObservation
	var recoveryBarrierActive bool
	deps.recover = func(ctx context.Context, _ config.Installation, _ PreflightResult, _ bool) error {
		recoveryContext = observeCleanupContext(ctx)
		recoveryBarrierActive = runner.maintenance
		runner.running, runner.coreRunning = true, true
		return nil
	}

	_, err := restoreWithDependencies(caller, installation, RestoreRequest{Archive: restoreArchive(t), Confirm: true}, deps)
	if !errors.Is(err, context.Canceled) {
		t.Fatalf("Restore() error = %v, want cancelled post-mutation restore", err)
	}
	if !recoveryBarrierActive {
		t.Fatal("post-mutation recovery started after the admission barrier was removed")
	}
	assertIndependentBoundedCleanupContexts(t, []cleanupContextObservation{recoveryContext})
	if runner.maintenance || !runner.running {
		t.Fatalf("post-mutation cancellation cleanup state = maintenance:%t running:%t", runner.maintenance, runner.running)
	}
}

func TestRestoreJoinsPrimaryRestartAndDeactivationFailures(t *testing.T) {
	installation := preflightTestInstallation(t)
	backing := newBackupRunner(installation, true)
	stopResponseLost := errors.New("stop response lost")
	startResponseLost := errors.New("restart response lost")
	deactivationResponseLost := errors.New("deactivation response lost")
	runner := &commandFailureRunner{
		fakeBackupRunner: backing,
		failures: []*commandFailure{
			{
				match: func(command string) bool { return strings.HasSuffix(command, " stop") },
				err:   stopResponseLost,
				effect: func() {
					backing.stopCount++
					backing.running, backing.coreRunning = false, false
				},
				remaining: 1,
			},
			{
				match: func(command string) bool { return strings.HasSuffix(command, " start") },
				err:   startResponseLost,
				effect: func() {
					backing.startCount++
					backing.running, backing.coreRunning = true, true
				},
				remaining: 1,
			},
			{
				match: func(command string) bool { return strings.Contains(command, " maintenance-deactivate") },
				err:   deactivationResponseLost,
				effect: func() {
					backing.maintenance = false
				},
				remaining: 1,
			},
		},
	}

	_, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: restoreArchive(t), Confirm: true}, restoreTestDependencies(t, runner))
	if !errors.Is(err, stopResponseLost) || !errors.Is(err, startResponseLost) || !errors.Is(err, deactivationResponseLost) {
		t.Fatalf("Restore() error = %v, want joined stop/restart/deactivation failures", err)
	}
	if backing.maintenance || !backing.running {
		t.Fatalf("response-loss cleanup state = maintenance:%t running:%t", backing.maintenance, backing.running)
	}
}

func TestRecoverRestoreTransactionVerifiesRecoveredStateBeforeReturning(t *testing.T) {
	installation := preflightTestInstallation(t)
	recovery, err := Preflight(context.Background(), installation, PreflightRequest{Archive: restoreArchive(t), Confirm: true, AllowExternalSecrets: true}, permissivePreflightDependencies())
	if err != nil {
		t.Fatal(err)
	}
	defer recovery.CloseArchive()
	runner := newBackupRunner(installation, true)
	deps := restoreTestDependencies(t, runner)
	var checks []string
	for _, name := range []string{"health", "doctor", "pi", "workspace"} {
		name := name
		deps.verify[name] = func(context.Context, config.Installation, archiveRunner) error {
			checks = append(checks, name)
			return nil
		}
	}

	if err := recoverRestoreTransaction(context.Background(), installation, recovery, true, deps); err != nil {
		t.Fatal(err)
	}
	if got, want := checks, []string{"health", "doctor", "pi", "workspace"}; !equalStrings(got, want) {
		t.Fatalf("recovery checks = %v, want %v", got, want)
	}
}

func TestRecoverRestoreTransactionFailsClosedForEveryVerification(t *testing.T) {
	for _, verification := range []string{"health", "doctor", "pi", "workspace"} {
		verification := verification
		t.Run(verification, func(t *testing.T) {
			installation := preflightTestInstallation(t)
			recovery, err := Preflight(context.Background(), installation, PreflightRequest{Archive: restoreArchive(t), Confirm: true, AllowExternalSecrets: true}, permissivePreflightDependencies())
			if err != nil {
				t.Fatal(err)
			}
			defer recovery.CloseArchive()
			runner := newBackupRunner(installation, true)
			runner.maintenance = true
			deps := restoreTestDependencies(t, runner)
			verificationErr := fmt.Errorf("recovery %s verification failed", verification)
			deps.verify[verification] = func(context.Context, config.Installation, archiveRunner) error {
				if !runner.maintenance {
					t.Fatal("recovery verification ran after the maintenance barrier was removed")
				}
				return verificationErr
			}

			err = recoverRestoreTransaction(context.Background(), installation, recovery, true, deps)
			if !errors.Is(err, verificationErr) {
				t.Fatalf("recoverRestoreTransaction() error = %v, want %v", err, verificationErr)
			}
			if !runner.maintenance {
				t.Fatal("recovery removed the maintenance barrier after a failed verification")
			}
		})
	}
}

func TestRestoreReleasesBarrierOnlyAfterVerifiedRecoveryFromLostResponse(t *testing.T) {
	for _, scenario := range []struct {
		name    string
		failure commandFailure
	}{
		{
			name: "recovery stop",
			failure: commandFailure{
				match: func(command string) bool { return strings.HasSuffix(command, " stop") },
				err:   errors.New("recovery stop response lost"),
				effect: func() {
					// The stop took effect before Docker lost the response.
				},
				skip:      1,
				remaining: 1,
			},
		},
		{
			name: "recovery start",
			failure: commandFailure{
				match: func(command string) bool { return strings.HasSuffix(command, " start") },
				err:   errors.New("recovery start response lost"),
				effect: func() {
					// The start took effect before Docker lost the response.
				},
				remaining: 1,
			},
		},
	} {
		scenario := scenario
		t.Run(scenario.name, func(t *testing.T) {
			installation := preflightTestInstallation(t)
			archive := restoreArchive(t)
			recovery, err := Preflight(context.Background(), installation, PreflightRequest{Archive: archive, Confirm: true, AllowExternalSecrets: true}, permissivePreflightDependencies())
			if err != nil {
				t.Fatal(err)
			}
			backing := newBackupRunner(installation, true)
			failure := scenario.failure
			originalEffect := failure.effect
			failure.effect = func() {
				if strings.Contains(scenario.name, "stop") {
					backing.stopCount++
					backing.running, backing.coreRunning = false, false
				} else {
					backing.startCount++
					backing.running, backing.coreRunning = true, true
				}
				if originalEffect != nil {
					originalEffect()
				}
			}
			runner := &commandFailureRunner{fakeBackupRunner: backing, failures: []*commandFailure{&failure}}
			deps := restoreTestDependencies(t, runner)
			deps.checkpoint = func(context.Context, config.Installation, CreateRequest) (Result, error) {
				return Result{Path: "/tmp/recovery.zip"}, nil
			}
			deps.prepareRecovery = func(context.Context, config.Installation, string) (PreflightResult, error) {
				return recovery, nil
			}
			restoreCalls := 0
			mutationErr := errors.New("target mutation failed")
			deps.restoreFile = func(context.Context, config.Installation, ArchiveEntryMetadata, io.Reader) error {
				restoreCalls++
				if restoreCalls == 1 {
					return mutationErr
				}
				return nil
			}
			deps.recover = func(ctx context.Context, target config.Installation, checkpoint PreflightResult, wasRunning bool) error {
				return recoverRestoreTransaction(ctx, target, checkpoint, wasRunning, deps)
			}

			_, err = restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: archive, Confirm: true}, deps)
			if !errors.Is(err, mutationErr) || !errors.Is(err, failure.err) {
				t.Fatalf("Restore() error = %v, want joined mutation and recovery response-loss errors", err)
			}
			if backing.maintenance || !backing.running {
				t.Fatalf("verified recovery state = maintenance:%t running:%t", backing.maintenance, backing.running)
			}
			assertIndependentBoundedCleanupContexts(t, runner.cleanupCommandContexts)
		})
	}
}

func TestRestoreRefusesActiveSessionsWithoutDrain(t *testing.T) {
	installation := preflightTestInstallation(t)
	archive := restoreArchive(t)
	runner := newBackupRunner(installation, true)
	runner.sessionResponses = []string{`[{"status":"running","archived":false}]`}
	deps := restoreTestDependencies(t, runner)
	restoredFiles := 0
	deps.restoreFile = func(context.Context, config.Installation, ArchiveEntryMetadata, io.Reader) error {
		restoredFiles++
		return nil
	}

	_, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: archive, Confirm: true}, deps)
	if !errors.Is(err, ErrActiveSessions) {
		t.Fatalf("restore error = %v, want active-session refusal", err)
	}
	if restoredFiles != 0 || runner.stopCount != 0 || runner.startCount != 0 || !runner.running || runner.maintenance {
		t.Fatalf("active-session refusal changed lifecycle state: files=%d stops=%d starts=%d running=%t maintenance=%t", restoredFiles, runner.stopCount, runner.startCount, runner.running, runner.maintenance)
	}
}

func TestRestoreCleansMaintenanceAfterActivationFailure(t *testing.T) {
	installation := preflightTestInstallation(t)
	backing := newBackupRunner(installation, true)
	activationErr := errors.New("activation response lost")
	runner := &restoreFailureRunner{
		fakeBackupRunner: backing,
		failContains:     "operator-command.js maintenance-activate",
		err:              activationErr,
		beforeFailure:    func() { backing.maintenance = true },
	}
	deps := restoreTestDependencies(t, runner)

	_, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: restoreArchive(t), Confirm: true}, deps)
	if !errors.Is(err, activationErr) {
		t.Fatalf("restore error = %v, want activation failure", err)
	}
	if backing.maintenance || !backing.running || runner.matchCount != 1 {
		t.Fatalf("activation cleanup state: maintenance=%t running=%t matches=%d", backing.maintenance, backing.running, runner.matchCount)
	}
}

func TestRestoreRestartsAndCleansMaintenanceAfterStopFailure(t *testing.T) {
	installation := preflightTestInstallation(t)
	backing := newBackupRunner(installation, true)
	stopErr := errors.New("stop response lost")
	runner := &restoreFailureRunner{
		fakeBackupRunner: backing,
		failSuffix:       " stop",
		err:              stopErr,
		beforeFailure: func() {
			backing.stopCount++
			backing.running, backing.coreRunning = false, false
		},
	}
	deps := restoreTestDependencies(t, runner)

	_, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: restoreArchive(t), Confirm: true}, deps)
	if !errors.Is(err, stopErr) {
		t.Fatalf("restore error = %v, want stop failure", err)
	}
	if backing.maintenance || !backing.running || backing.startCount != 1 {
		t.Fatalf("stop cleanup state: maintenance=%t running=%t starts=%d", backing.maintenance, backing.running, backing.startCount)
	}
}

func TestRestoreCleansMaintenanceAfterMutationAndRollbackFailures(t *testing.T) {
	installation := preflightTestInstallation(t)
	for _, test := range []struct {
		name        string
		recoveryErr error
	}{
		{name: "mutation"},
		{name: "rollback", recoveryErr: errors.New("rollback failed")},
	} {
		t.Run(test.name, func(t *testing.T) {
			backing := newBackupRunner(installation, true)
			deps := restoreTestDependencies(t, backing)
			mutationErr := errors.New("mutation failed")
			deps.restoreFile = func(context.Context, config.Installation, ArchiveEntryMetadata, io.Reader) error {
				return mutationErr
			}
			deps.recover = func(context.Context, config.Installation, PreflightResult, bool) error {
				if test.recoveryErr == nil {
					backing.running, backing.coreRunning = true, true
				}
				return test.recoveryErr
			}

			_, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: restoreArchive(t), Confirm: true}, deps)
			if !errors.Is(err, mutationErr) || (test.recoveryErr != nil && !errors.Is(err, test.recoveryErr)) {
				t.Fatalf("restore error = %v, want mutation and rollback failures", err)
			}
			if test.recoveryErr == nil && (backing.maintenance || !backing.running) {
				t.Fatalf("successful recovery cleanup state: maintenance=%t running=%t", backing.maintenance, backing.running)
			}
			if test.recoveryErr != nil && !backing.maintenance {
				t.Fatal("failed recovery removed the maintenance barrier before a verified rollback")
			}
		})
	}
}

func TestRestorePreservesDrainAndMaintenanceCleanupFailures(t *testing.T) {
	installation := preflightTestInstallation(t)
	backing := newBackupRunner(installation, true)
	backing.sessionResponses = []string{`[{"status":"running","archived":false}]`}
	cleanupErr := errors.New("maintenance cleanup failed")
	runner := &restoreFailureRunner{
		fakeBackupRunner: backing,
		failContains:     "operator-command.js maintenance-deactivate",
		err:              cleanupErr,
		beforeFailure:    func() { backing.maintenance = false },
	}
	deps := restoreTestDependencies(t, runner)

	_, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: restoreArchive(t), Confirm: true}, deps)
	if !errors.Is(err, ErrActiveSessions) || !errors.Is(err, cleanupErr) {
		t.Fatalf("restore error = %v, want drain and cleanup failures", err)
	}
	if backing.maintenance || !backing.running {
		t.Fatalf("cleanup failure state: maintenance=%t running=%t", backing.maintenance, backing.running)
	}
}

func restoreArchive(t *testing.T) string {
	t.Helper()
	archive := filepath.Join(t.TempDir(), "restore.zip")
	writePreflightArchive(t, archive, preflightArchiveSpec{
		entries: []preflightArchiveEntry{{path: "configuration/operator.env", body: []byte("safe")}},
	})
	return archive
}

type failStartRestoreRunner struct {
	*fakeBackupRunner
}

func (runner failStartRestoreRunner) Run(ctx context.Context, args []string, stdin io.Reader) (compose.Result, error) {
	if strings.HasSuffix(strings.Join(args, " "), " start") {
		return compose.Result{}, errors.New("start refused")
	}
	return runner.fakeBackupRunner.Run(ctx, args, stdin)
}

type restoreFailureRunner struct {
	*fakeBackupRunner
	failContains  string
	failSuffix    string
	err           error
	beforeFailure func()
	matchCount    int
}

func (runner *restoreFailureRunner) Run(ctx context.Context, args []string, stdin io.Reader) (compose.Result, error) {
	command := strings.Join(args, " ")
	matches := runner.failContains != "" && strings.Contains(command, runner.failContains)
	matches = matches || runner.failSuffix != "" && strings.HasSuffix(command, runner.failSuffix)
	if matches {
		runner.matchCount++
		if runner.beforeFailure != nil {
			runner.beforeFailure()
		}
		return compose.Result{}, runner.err
	}
	return runner.fakeBackupRunner.Run(ctx, args, stdin)
}

func (runner *restoreFailureRunner) Stream(ctx context.Context, args []string, stdin io.Reader, stdout io.Writer) (compose.Result, error) {
	return runner.fakeBackupRunner.Stream(ctx, args, stdin, stdout)
}

func (runner *restoreFailureRunner) SessionInventoryScope() string {
	return runner.fakeBackupRunner.SessionInventoryScope()
}

type fakeRestoreLock struct {
	release func()
}

type authenticationStateResetRunner struct {
	args   []string
	result compose.Result
	err    error
}

func (runner *authenticationStateResetRunner) Run(_ context.Context, args []string, _ io.Reader) (compose.Result, error) {
	runner.args = append([]string(nil), args...)
	return runner.result, runner.err
}

func (runner *authenticationStateResetRunner) Stream(context.Context, []string, io.Reader, io.Writer) (compose.Result, error) {
	return compose.Result{}, errors.New("authentication state reset must not stream a volume archive")
}

func (*authenticationStateResetRunner) SessionInventoryScope() string { return "mine" }

type workspaceVerificationRunner struct {
	running bool
	result  compose.Result
	err     error
	calls   []string
}

func (runner *workspaceVerificationRunner) Run(_ context.Context, args []string, _ io.Reader) (compose.Result, error) {
	command := strings.Join(args, " ")
	runner.calls = append(runner.calls, command)
	if strings.Contains(command, " ps --all --format json") {
		if runner.running {
			return compose.Result{Stdout: healthyServicesPayload()}, nil
		}
		return compose.Result{}, nil
	}
	if strings.Contains(command, "operator-command.js workspace-integrity") {
		return runner.result, runner.err
	}
	return compose.Result{}, fmt.Errorf("unexpected workspace verification command: %s", command)
}

func (*workspaceVerificationRunner) Stream(context.Context, []string, io.Reader, io.Writer) (compose.Result, error) {
	return compose.Result{}, errors.New("workspace verification must not stream")
}

func (*workspaceVerificationRunner) SessionInventoryScope() string { return "mine" }

func TestVerifyRestoreWorkspaceUsesFixedNonNetworkOperatorPath(t *testing.T) {
	installation := preflightTestInstallation(t)
	fingerprint := "sha256:" + strings.Repeat("a", 64)
	for _, test := range []struct {
		name    string
		running bool
		payload string
		prefix  string
	}{
		{name: "stopped uninitialized", payload: fmt.Sprintf(`{"ready":true,"state":"uninitialized","workspaces":0,"fingerprint":%q}`, fingerprint), prefix: "run --rm --no-deps --no-TTY core"},
		{name: "running active", running: true, payload: fmt.Sprintf(`{"ready":true,"state":"active","workspaces":1,"fingerprint":%q}`, fingerprint), prefix: "exec -T core"},
	} {
		t.Run(test.name, func(t *testing.T) {
			runner := &workspaceVerificationRunner{
				running: test.running,
				result:  compose.Result{Stdout: test.payload},
			}
			if err := verifyRestoreWorkspace(context.Background(), installation, runner); err != nil {
				t.Fatalf("verify restored workspace: %v", err)
			}
			if len(runner.calls) != 2 || !strings.Contains(runner.calls[1], test.prefix+" node /app/backend/dist/operator-command.js workspace-integrity") {
				t.Fatalf("workspace verification calls = %#v", runner.calls)
			}
			for _, call := range runner.calls {
				if strings.Contains(call, "curl") || strings.Contains(call, "-e const") || strings.Contains(strings.ToLower(call), "header") {
					t.Fatalf("workspace verification used an unsafe command: %s", call)
				}
			}
		})
	}
}

func TestVerifyRestoreWorkspaceRejectsInvalidOperatorResults(t *testing.T) {
	installation := preflightTestInstallation(t)
	valid := `{"ready":true,"state":"active","workspaces":1,"fingerprint":"sha256:` + strings.Repeat("a", 64) + `"}`
	for _, test := range []struct {
		name   string
		result compose.Result
		err    error
	}{
		{name: "empty"},
		{name: "malformed", result: compose.Result{Stdout: `{malformed`}},
		{name: "trailing document", result: compose.Result{Stdout: valid + `{}`}},
		{name: "unknown field", result: compose.Result{Stdout: strings.TrimSuffix(valid, "}") + `,"detail":"unsafe"}`}},
		{name: "not ready", result: compose.Result{Stdout: strings.Replace(valid, `"ready":true`, `"ready":false`, 1)}},
		{name: "unknown state", result: compose.Result{Stdout: strings.Replace(valid, `"state":"active"`, `"state":"unknown"`, 1)}},
		{name: "inconsistent count", result: compose.Result{Stdout: strings.Replace(valid, `"state":"active"`, `"state":"uninitialized"`, 1)}},
		{name: "missing fingerprint", result: compose.Result{Stdout: `{"ready":true,"state":"active","workspaces":1}`}},
		{name: "malformed fingerprint", result: compose.Result{Stdout: `{"ready":true,"state":"active","workspaces":1,"fingerprint":"sha256:not-a-digest"}`}},
		{name: "nonzero", result: compose.Result{ExitCode: 2}, err: errors.New("exit status 2")},
	} {
		t.Run(test.name, func(t *testing.T) {
			runner := &workspaceVerificationRunner{result: test.result, err: test.err}
			if err := verifyRestoreWorkspace(context.Background(), installation, runner); err == nil {
				t.Fatal("invalid workspace verification result was accepted")
			}
		})
	}
}

func (lock fakeRestoreLock) Release() error {
	if lock.release != nil {
		lock.release()
	}
	return nil
}

func equalStrings(got, want []string) bool {
	if len(got) != len(want) {
		return false
	}
	for index := range got {
		if got[index] != want[index] {
			return false
		}
	}
	return true
}

func restoreTestDependencies(t *testing.T, runner archiveRunner) restoreDependencies {
	t.Helper()
	return restoreDependencies{
		preflight: func(ctx context.Context, installation config.Installation, request PreflightRequest) (PreflightResult, error) {
			return Preflight(ctx, installation, request, permissivePreflightDependencies())
		},
		checkpoint: func(context.Context, config.Installation, CreateRequest) (Result, error) {
			return Result{Path: "/tmp/default-checkpoint.zip"}, nil
		},
		prepareRecovery: func(context.Context, config.Installation, string) (PreflightResult, error) {
			return PreflightResult{}, nil
		},
		recover:           func(context.Context, config.Installation, PreflightResult, bool) error { return nil },
		cleanupCheckpoint: func(string) error { return nil },
		acquireLock:       func(config.Installation) (restoreLock, error) { return fakeRestoreLock{}, nil },
		runner:            runner,
		sleep:             func(time.Duration) {},
		restoreFile:       func(context.Context, config.Installation, ArchiveEntryMetadata, io.Reader) error { return nil },
		restoreVolume: func(context.Context, config.Installation, VolumeMetadata, io.Reader) error {
			return nil
		},
		resetAuthenticationState: func(context.Context, config.Installation, archiveRunner) error {
			return nil
		},
		verify: map[string]restoreVerify{
			"health":    func(context.Context, config.Installation, archiveRunner) error { return nil },
			"doctor":    func(context.Context, config.Installation, archiveRunner) error { return nil },
			"pi":        func(context.Context, config.Installation, archiveRunner) error { return nil },
			"workspace": func(context.Context, config.Installation, archiveRunner) error { return nil },
		},
	}
}