650 lines
26 KiB
Go
650 lines
26 KiB
Go
package backup
|
|
|
|
import (
|
|
"archive/tar"
|
|
"bytes"
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/aritmolab/thothii/tools/tht/internal/compose"
|
|
"github.com/aritmolab/thothii/tools/tht/internal/config"
|
|
)
|
|
|
|
func TestRestorePublicPathUsesConcreteProductionPreflight(t *testing.T) {
|
|
root := t.TempDir()
|
|
installation := config.Installation{
|
|
Path: filepath.Join(root, "deploy", "local-dev", "thothii-installation.yaml"),
|
|
ProjectDirectory: root,
|
|
}
|
|
missing := filepath.Join(root, "missing.zip")
|
|
|
|
_, err := Restore(context.Background(), installation, RestoreRequest{Archive: missing, Confirm: true})
|
|
|
|
if err == nil || strings.Contains(err.Error(), "dependencies are unavailable") || !strings.Contains(err.Error(), "backup archive") {
|
|
t.Fatalf("Restore() error = %v, want production archive preflight", err)
|
|
}
|
|
}
|
|
|
|
func TestVolumeRestoreCommandKeepsTarInputOpenWithoutTTY(t *testing.T) {
|
|
args := volumeRestoreCommand("project_sessions")
|
|
wantPrefix := []string{"run", "--rm", "--interactive", "--network", "none"}
|
|
if len(args) < len(wantPrefix) || !equalStrings(args[:len(wantPrefix)], wantPrefix) {
|
|
t.Fatalf("volumeRestoreCommand() prefix = %q, want %q", args, wantPrefix)
|
|
}
|
|
for _, arg := range args {
|
|
if arg == "--tty" || arg == "-t" {
|
|
t.Fatalf("volumeRestoreCommand() requests a TTY: %q", args)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestRestoreRestoresVerifiedVolumesInManifestOrderBeforeAuthenticationReset(t *testing.T) {
|
|
installation := preflightTestInstallation(t)
|
|
archive := filepath.Join(t.TempDir(), "restore-volumes.zip")
|
|
sessionsTar := safeRestoreTar(t, "session.txt", "session")
|
|
settingsTar := safeRestoreTar(t, "settings.json", "settings")
|
|
writePreflightArchive(t, archive, preflightArchiveSpec{
|
|
volumes: []VolumeMetadata{
|
|
{LogicalName: "sessions", Name: "project_sessions", Driver: "local"},
|
|
{LogicalName: "settings", Name: "project_settings", Driver: "local"},
|
|
},
|
|
entries: []preflightArchiveEntry{
|
|
{path: "configuration/operator.env", body: []byte("safe")},
|
|
{path: "volumes/settings.tar", body: settingsTar, kind: EntryVolume, owner: "volume:settings", logicalName: "settings"},
|
|
{path: "volumes/sessions.tar", body: sessionsTar, kind: EntryVolume, owner: "volume:sessions", logicalName: "sessions"},
|
|
},
|
|
})
|
|
runner := newBackupRunner(installation, false)
|
|
deps := restoreTestDependencies(t, runner)
|
|
var events []string
|
|
deps.restoreFile = func(_ context.Context, _ config.Installation, entry ArchiveEntryMetadata, _ io.Reader) error {
|
|
events = append(events, "file:"+entry.Path)
|
|
return nil
|
|
}
|
|
deps.restoreVolume = func(_ context.Context, _ config.Installation, volume VolumeMetadata, stream io.Reader) error {
|
|
if _, err := io.ReadAll(stream); err != nil {
|
|
return err
|
|
}
|
|
events = append(events, "volume:"+volume.LogicalName)
|
|
return nil
|
|
}
|
|
deps.resetAuthenticationState = func(context.Context, config.Installation, archiveRunner) error {
|
|
events = append(events, "reset-auth-state")
|
|
return nil
|
|
}
|
|
|
|
if _, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: archive, Confirm: true}, deps); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got, want := events, []string{
|
|
"file:configuration/operator.env",
|
|
"volume:sessions",
|
|
"volume:settings",
|
|
"reset-auth-state",
|
|
}; !equalStrings(got, want) {
|
|
t.Fatalf("restore events = %v, want %v", got, want)
|
|
}
|
|
}
|
|
|
|
func safeRestoreTar(t *testing.T, name, contents string) []byte {
|
|
t.Helper()
|
|
var output bytes.Buffer
|
|
writer := tar.NewWriter(&output)
|
|
if err := writer.WriteHeader(&tar.Header{Name: name, Mode: 0o600, Size: int64(len(contents)), Typeflag: tar.TypeReg}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := writer.Write([]byte(contents)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := writer.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return output.Bytes()
|
|
}
|
|
|
|
func TestRestoreStoppedInstallationRunsCheckpointRestoreAndVerification(t *testing.T) {
|
|
installation := preflightTestInstallation(t)
|
|
archive := filepath.Join(t.TempDir(), "restore.zip")
|
|
writePreflightArchive(t, archive, preflightArchiveSpec{
|
|
entries: []preflightArchiveEntry{{path: "configuration/operator.env", body: []byte("safe")}},
|
|
})
|
|
runner := newBackupRunner(installation, false)
|
|
var events []string
|
|
var checkpointRequest CreateRequest
|
|
deps := restoreTestDependencies(t, runner)
|
|
deps.checkpoint = func(_ context.Context, _ config.Installation, request CreateRequest) (Result, error) {
|
|
events = append(events, "checkpoint")
|
|
checkpointRequest = request
|
|
return Result{Path: "/tmp/checkpoint.zip"}, nil
|
|
}
|
|
deps.prepareRecovery = func(context.Context, config.Installation, string) (PreflightResult, error) {
|
|
events = append(events, "prepare-recovery")
|
|
return PreflightResult{}, nil
|
|
}
|
|
deps.cleanupCheckpoint = func(string) error {
|
|
events = append(events, "cleanup-checkpoint")
|
|
return nil
|
|
}
|
|
deps.acquireLock = func(config.Installation) (restoreLock, error) {
|
|
events = append(events, "lock")
|
|
return fakeRestoreLock{release: func() { events = append(events, "unlock") }}, nil
|
|
}
|
|
deps.restoreFile = func(_ context.Context, _ config.Installation, entry ArchiveEntryMetadata, _ io.Reader) error {
|
|
events = append(events, "file:"+entry.Path)
|
|
return nil
|
|
}
|
|
for _, name := range []string{"health", "doctor", "pi", "workspace"} {
|
|
name := name
|
|
deps.verify[name] = func(context.Context, config.Installation, archiveRunner) error {
|
|
events = append(events, name)
|
|
return nil
|
|
}
|
|
}
|
|
|
|
result, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: archive, Confirm: true}, deps)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if result.Checkpoint != "" || result.Restarted || !result.Verified {
|
|
t.Fatalf("Restore() result = %#v", result)
|
|
}
|
|
if !checkpointRequest.IncludeSecrets || !checkpointRequest.Confirm {
|
|
t.Fatalf("checkpoint request = %#v, want private confirmed secret-aware checkpoint", checkpointRequest)
|
|
}
|
|
if got, want := events, []string{"checkpoint", "prepare-recovery", "lock", "file:configuration/operator.env", "health", "doctor", "pi", "workspace", "unlock", "cleanup-checkpoint"}; !equalStrings(got, want) {
|
|
t.Fatalf("restore events = %v, want %v", got, want)
|
|
}
|
|
}
|
|
|
|
func TestRestoreResetsAuthenticationStateBeforeRestart(t *testing.T) {
|
|
installation := preflightTestInstallation(t)
|
|
archive := restoreArchive(t)
|
|
runner := newBackupRunner(installation, true)
|
|
deps := restoreTestDependencies(t, runner)
|
|
var events []string
|
|
deps.restoreFile = func(_ context.Context, _ config.Installation, entry ArchiveEntryMetadata, _ io.Reader) error {
|
|
events = append(events, "file:"+entry.Path)
|
|
return nil
|
|
}
|
|
deps.resetAuthenticationState = func(context.Context, config.Installation, archiveRunner) error {
|
|
events = append(events, "reset-auth-state")
|
|
return nil
|
|
}
|
|
for _, name := range []string{"health", "doctor", "pi", "workspace"} {
|
|
name := name
|
|
deps.verify[name] = func(context.Context, config.Installation, archiveRunner) error {
|
|
events = append(events, name)
|
|
return nil
|
|
}
|
|
}
|
|
|
|
result, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: archive, Confirm: true}, deps)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !result.Restarted || !result.Verified {
|
|
t.Fatalf("restore result = %#v", result)
|
|
}
|
|
if got, want := events, []string{"file:configuration/operator.env", "reset-auth-state", "health", "doctor", "pi", "workspace"}; !equalStrings(got, want) {
|
|
t.Fatalf("restore events = %v, want %v", got, want)
|
|
}
|
|
}
|
|
|
|
func TestResetAuthenticationStateCreatesOnlyPrivateEmptyStateDirectories(t *testing.T) {
|
|
installation := preflightTestInstallation(t)
|
|
runner := &authenticationStateResetRunner{}
|
|
|
|
if err := resetAuthenticationState(context.Background(), installation, runner); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
joined := strings.Join(runner.args, "\x00")
|
|
for _, required := range []string{
|
|
"run", "--rm", "--no-deps", "--no-TTY", "--user", "0:0", "--entrypoint", "sh", "core", "-ceu",
|
|
"test ! -L /data/auth",
|
|
"install -d -o 10001 -g 10001 -m 0700 /data/auth",
|
|
"find /data/auth -mindepth 1 -maxdepth 1 -exec rm -rf -- {} +",
|
|
"install -d -o 10001 -g 10001 -m 0700 /data/auth/sessions /data/auth/oidc",
|
|
"find /data/auth/sessions /data/auth/oidc -mindepth 1 -print -quit",
|
|
} {
|
|
if !strings.Contains(joined, required) {
|
|
t.Fatalf("authentication state reset command omits %q: %#v", required, runner.args)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestResetAuthenticationStateFailsClosedForDamagedRoot(t *testing.T) {
|
|
installation := preflightTestInstallation(t)
|
|
runner := &authenticationStateResetRunner{result: compose.Result{ExitCode: 1}, err: errors.New("damaged auth root")}
|
|
|
|
err := resetAuthenticationState(context.Background(), installation, runner)
|
|
if err == nil || !strings.Contains(err.Error(), "reset authentication state") {
|
|
t.Fatalf("resetAuthenticationState() error = %v, want bounded fail-closed error", err)
|
|
}
|
|
}
|
|
|
|
func TestRestorePreflightFailureDoesNotMutateTarget(t *testing.T) {
|
|
installation := preflightTestInstallation(t)
|
|
runner := newBackupRunner(installation, true)
|
|
deps := restoreTestDependencies(t, runner)
|
|
preflightErr := errors.New("archive cannot be restored")
|
|
deps.preflight = func(context.Context, config.Installation, PreflightRequest) (PreflightResult, error) {
|
|
return PreflightResult{}, preflightErr
|
|
}
|
|
checkpointCalls := 0
|
|
deps.checkpoint = func(context.Context, config.Installation, CreateRequest) (Result, error) {
|
|
checkpointCalls++
|
|
return Result{}, nil
|
|
}
|
|
restoredFiles := 0
|
|
deps.restoreFile = func(context.Context, config.Installation, ArchiveEntryMetadata, io.Reader) error {
|
|
restoredFiles++
|
|
return nil
|
|
}
|
|
|
|
result, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: "unreadable.zip", Confirm: true}, deps)
|
|
if !errors.Is(err, preflightErr) {
|
|
t.Fatalf("restore error = %v, want preflight error", err)
|
|
}
|
|
if result != (RestoreResult{}) || checkpointCalls != 0 || restoredFiles != 0 || runner.stopCount != 0 || runner.startCount != 0 || !runner.running {
|
|
t.Fatalf("preflight failure mutated target: result=%#v checkpoint=%d files=%d stops=%d starts=%d running=%t", result, checkpointCalls, restoredFiles, runner.stopCount, runner.startCount, runner.running)
|
|
}
|
|
}
|
|
|
|
func TestRestoreCheckpointFailureDoesNotMutateTarget(t *testing.T) {
|
|
installation := preflightTestInstallation(t)
|
|
archive := restoreArchive(t)
|
|
runner := newBackupRunner(installation, true)
|
|
deps := restoreTestDependencies(t, runner)
|
|
checkpointErr := errors.New("checkpoint unavailable")
|
|
deps.checkpoint = func(context.Context, config.Installation, CreateRequest) (Result, error) {
|
|
return Result{}, checkpointErr
|
|
}
|
|
restoredFiles := 0
|
|
deps.restoreFile = func(context.Context, config.Installation, ArchiveEntryMetadata, io.Reader) error {
|
|
restoredFiles++
|
|
return nil
|
|
}
|
|
|
|
result, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: archive, Confirm: true}, deps)
|
|
if !errors.Is(err, checkpointErr) {
|
|
t.Fatalf("restore error = %v, want checkpoint error", err)
|
|
}
|
|
if result != (RestoreResult{}) || restoredFiles != 0 || runner.stopCount != 0 || runner.startCount != 0 || !runner.running {
|
|
t.Fatalf("checkpoint failure mutated target: result=%#v files=%d stops=%d starts=%d running=%t", result, restoredFiles, runner.stopCount, runner.startCount, runner.running)
|
|
}
|
|
}
|
|
|
|
func TestRestoreFileFailureRollsBackSecretAwareCheckpointBeforeCleanup(t *testing.T) {
|
|
installation := preflightTestInstallation(t)
|
|
archive := restoreArchive(t)
|
|
runner := newBackupRunner(installation, true)
|
|
deps := restoreTestDependencies(t, runner)
|
|
fileErr := errors.New("cannot restore operator configuration")
|
|
deps.checkpoint = func(context.Context, config.Installation, CreateRequest) (Result, error) {
|
|
return Result{Path: "/tmp/recovery.zip"}, nil
|
|
}
|
|
var events []string
|
|
deps.recover = func(context.Context, config.Installation, PreflightResult, bool) error {
|
|
events = append(events, "recover")
|
|
return nil
|
|
}
|
|
deps.cleanupCheckpoint = func(string) error {
|
|
events = append(events, "cleanup")
|
|
return nil
|
|
}
|
|
deps.restoreFile = func(context.Context, config.Installation, ArchiveEntryMetadata, io.Reader) error {
|
|
events = append(events, "mutate")
|
|
return fileErr
|
|
}
|
|
|
|
result, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: archive, Confirm: true}, deps)
|
|
if !errors.Is(err, fileErr) {
|
|
t.Fatalf("restore error = %v, want file error", err)
|
|
}
|
|
if result.Checkpoint != "" {
|
|
t.Fatalf("recovery checkpoint = %q, want no retained secret-bearing path", result.Checkpoint)
|
|
}
|
|
if got, want := events, []string{"mutate", "recover", "cleanup"}; !equalStrings(got, want) {
|
|
t.Fatalf("failure recovery events = %v, want %v", got, want)
|
|
}
|
|
}
|
|
|
|
func TestRestoreFailureAfterAuthenticationMutationRollsBackAndClearsRuntimeState(t *testing.T) {
|
|
installation := preflightTestInstallation(t)
|
|
archive := restoreArchive(t)
|
|
runner := newBackupRunner(installation, false)
|
|
deps := restoreTestDependencies(t, runner)
|
|
resetErr := errors.New("authentication reset failed")
|
|
var events []string
|
|
deps.restoreFile = func(context.Context, config.Installation, ArchiveEntryMetadata, io.Reader) error {
|
|
events = append(events, "auth-config-mutated")
|
|
return nil
|
|
}
|
|
deps.resetAuthenticationState = func(context.Context, config.Installation, archiveRunner) error {
|
|
events = append(events, "auth-runtime-reset-failed")
|
|
return resetErr
|
|
}
|
|
deps.recover = func(context.Context, config.Installation, PreflightResult, bool) error {
|
|
events = append(events, "secret-aware-recovery-and-reauth-reset")
|
|
return nil
|
|
}
|
|
deps.cleanupCheckpoint = func(string) error {
|
|
events = append(events, "checkpoint-cleaned")
|
|
return nil
|
|
}
|
|
|
|
_, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: archive, Confirm: true}, deps)
|
|
if !errors.Is(err, resetErr) {
|
|
t.Fatalf("restore error = %v, want reset failure", err)
|
|
}
|
|
if got, want := events, []string{"auth-config-mutated", "auth-runtime-reset-failed", "secret-aware-recovery-and-reauth-reset", "checkpoint-cleaned"}; !equalStrings(got, want) {
|
|
t.Fatalf("post-auth recovery events = %v, want %v", got, want)
|
|
}
|
|
}
|
|
|
|
func TestRestoreCleanupFailureDoesNotSuppressRollback(t *testing.T) {
|
|
installation := preflightTestInstallation(t)
|
|
archive := restoreArchive(t)
|
|
deps := restoreTestDependencies(t, newBackupRunner(installation, false))
|
|
mutationErr := errors.New("post-mutation failure")
|
|
cleanupErr := errors.New("checkpoint cleanup failure")
|
|
recovered := false
|
|
deps.restoreFile = func(context.Context, config.Installation, ArchiveEntryMetadata, io.Reader) error { return mutationErr }
|
|
deps.recover = func(context.Context, config.Installation, PreflightResult, bool) error { recovered = true; return nil }
|
|
deps.cleanupCheckpoint = func(string) error { return cleanupErr }
|
|
|
|
_, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: archive, Confirm: true}, deps)
|
|
if !recovered || !errors.Is(err, mutationErr) || !errors.Is(err, cleanupErr) {
|
|
t.Fatalf("restore error = %v recovered=%t, want joined mutation/cleanup failure after rollback", err, recovered)
|
|
}
|
|
}
|
|
|
|
func TestRecoveryCheckpointCleanupRejectsSymlinksAndPermissiveFiles(t *testing.T) {
|
|
root := t.TempDir()
|
|
target := filepath.Join(root, "checkpoint.zip")
|
|
if err := os.WriteFile(target, []byte("secret"), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
link := filepath.Join(root, "checkpoint-link.zip")
|
|
if err := os.Symlink(target, link); err != nil {
|
|
t.Skipf("symlink unavailable: %v", err)
|
|
}
|
|
if err := cleanupRecoveryCheckpoint(link); err == nil {
|
|
t.Fatal("cleanupRecoveryCheckpoint accepted a symlink")
|
|
}
|
|
if _, err := os.Stat(target); err != nil {
|
|
t.Fatalf("symlink target was changed: %v", err)
|
|
}
|
|
if err := os.Chmod(target, 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := cleanupRecoveryCheckpoint(target); err == nil {
|
|
t.Fatal("cleanupRecoveryCheckpoint accepted a permissive secret checkpoint")
|
|
}
|
|
}
|
|
|
|
func TestRecoveryCheckpointCleanupRemovesOnlyPrivateRegularFile(t *testing.T) {
|
|
root, err := filepath.EvalSymlinks(t.TempDir())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
checkpoint := filepath.Join(root, "checkpoint.zip")
|
|
if err := os.WriteFile(checkpoint, []byte("secret checkpoint"), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := cleanupRecoveryCheckpoint(checkpoint); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := os.Lstat(checkpoint); !errors.Is(err, os.ErrNotExist) {
|
|
t.Fatalf("private checkpoint still exists after cleanup: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestRestoreStartFailureRecoversPreviouslyRunningTarget(t *testing.T) {
|
|
installation := preflightTestInstallation(t)
|
|
archive := restoreArchive(t)
|
|
backingRunner := newBackupRunner(installation, true)
|
|
deps := restoreTestDependencies(t, failStartRestoreRunner{fakeBackupRunner: backingRunner})
|
|
recovered := false
|
|
deps.recover = func(context.Context, config.Installation, PreflightResult, bool) error {
|
|
recovered = true
|
|
backingRunner.running = true
|
|
return nil
|
|
}
|
|
|
|
_, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: archive, Confirm: true}, deps)
|
|
if err == nil || !strings.Contains(err.Error(), "start refused") {
|
|
t.Fatalf("restore error = %v, want restart failure", err)
|
|
}
|
|
if !recovered || !backingRunner.running {
|
|
t.Fatalf("failed restart was not rolled back: recovered=%t running=%t", recovered, backingRunner.running)
|
|
}
|
|
}
|
|
|
|
func TestRestoreVerificationFailureRecoversPreviouslyRunningTarget(t *testing.T) {
|
|
installation := preflightTestInstallation(t)
|
|
archive := restoreArchive(t)
|
|
runner := newBackupRunner(installation, true)
|
|
deps := restoreTestDependencies(t, runner)
|
|
verificationErr := errors.New("Pi is unavailable")
|
|
deps.verify["pi"] = func(context.Context, config.Installation, archiveRunner) error { return verificationErr }
|
|
recovered := false
|
|
deps.recover = func(context.Context, config.Installation, PreflightResult, bool) error {
|
|
recovered = true
|
|
return nil
|
|
}
|
|
|
|
_, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: archive, Confirm: true}, deps)
|
|
if !errors.Is(err, verificationErr) {
|
|
t.Fatalf("restore error = %v, want verification failure", err)
|
|
}
|
|
if !recovered || !runner.running {
|
|
t.Fatalf("verification failure was not rolled back: recovered=%t running=%t", recovered, runner.running)
|
|
}
|
|
}
|
|
|
|
func TestRestoreRefusesActiveSessionsWithoutDrain(t *testing.T) {
|
|
installation := preflightTestInstallation(t)
|
|
archive := restoreArchive(t)
|
|
runner := newBackupRunner(installation, true)
|
|
runner.sessionResponses = []string{`[{"status":"running","archived":false}]`}
|
|
deps := restoreTestDependencies(t, runner)
|
|
restoredFiles := 0
|
|
deps.restoreFile = func(context.Context, config.Installation, ArchiveEntryMetadata, io.Reader) error {
|
|
restoredFiles++
|
|
return nil
|
|
}
|
|
|
|
_, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: archive, Confirm: true}, deps)
|
|
if !errors.Is(err, ErrActiveSessions) {
|
|
t.Fatalf("restore error = %v, want active-session refusal", err)
|
|
}
|
|
if restoredFiles != 0 || runner.stopCount != 0 || runner.startCount != 0 || !runner.running {
|
|
t.Fatalf("active-session refusal mutated target: files=%d stops=%d starts=%d running=%t", restoredFiles, runner.stopCount, runner.startCount, runner.running)
|
|
}
|
|
}
|
|
|
|
func restoreArchive(t *testing.T) string {
|
|
t.Helper()
|
|
archive := filepath.Join(t.TempDir(), "restore.zip")
|
|
writePreflightArchive(t, archive, preflightArchiveSpec{
|
|
entries: []preflightArchiveEntry{{path: "configuration/operator.env", body: []byte("safe")}},
|
|
})
|
|
return archive
|
|
}
|
|
|
|
type failStartRestoreRunner struct {
|
|
*fakeBackupRunner
|
|
}
|
|
|
|
func (runner failStartRestoreRunner) Run(ctx context.Context, args []string, stdin io.Reader) (compose.Result, error) {
|
|
if strings.HasSuffix(strings.Join(args, " "), " start") {
|
|
return compose.Result{}, errors.New("start refused")
|
|
}
|
|
return runner.fakeBackupRunner.Run(ctx, args, stdin)
|
|
}
|
|
|
|
type fakeRestoreLock struct {
|
|
release func()
|
|
}
|
|
|
|
type authenticationStateResetRunner struct {
|
|
args []string
|
|
result compose.Result
|
|
err error
|
|
}
|
|
|
|
func (runner *authenticationStateResetRunner) Run(_ context.Context, args []string, _ io.Reader) (compose.Result, error) {
|
|
runner.args = append([]string(nil), args...)
|
|
return runner.result, runner.err
|
|
}
|
|
|
|
func (runner *authenticationStateResetRunner) Stream(context.Context, []string, io.Reader, io.Writer) (compose.Result, error) {
|
|
return compose.Result{}, errors.New("authentication state reset must not stream a volume archive")
|
|
}
|
|
|
|
func (*authenticationStateResetRunner) SessionInventoryScope() string { return "mine" }
|
|
|
|
type workspaceVerificationRunner struct {
|
|
running bool
|
|
result compose.Result
|
|
err error
|
|
calls []string
|
|
}
|
|
|
|
func (runner *workspaceVerificationRunner) Run(_ context.Context, args []string, _ io.Reader) (compose.Result, error) {
|
|
command := strings.Join(args, " ")
|
|
runner.calls = append(runner.calls, command)
|
|
if strings.Contains(command, " ps --all --format json") {
|
|
if runner.running {
|
|
return compose.Result{Stdout: healthyServicesPayload()}, nil
|
|
}
|
|
return compose.Result{}, nil
|
|
}
|
|
if strings.Contains(command, "operator-command.js workspace-integrity") {
|
|
return runner.result, runner.err
|
|
}
|
|
return compose.Result{}, fmt.Errorf("unexpected workspace verification command: %s", command)
|
|
}
|
|
|
|
func (*workspaceVerificationRunner) Stream(context.Context, []string, io.Reader, io.Writer) (compose.Result, error) {
|
|
return compose.Result{}, errors.New("workspace verification must not stream")
|
|
}
|
|
|
|
func (*workspaceVerificationRunner) SessionInventoryScope() string { return "mine" }
|
|
|
|
func TestVerifyRestoreWorkspaceUsesFixedNonNetworkOperatorPath(t *testing.T) {
|
|
installation := preflightTestInstallation(t)
|
|
for _, test := range []struct {
|
|
name string
|
|
running bool
|
|
payload string
|
|
prefix string
|
|
}{
|
|
{name: "stopped uninitialized", payload: `{"ready":true,"state":"uninitialized","workspaces":0}`, prefix: "run --rm --no-deps --no-TTY core"},
|
|
{name: "running active", running: true, payload: `{"ready":true,"state":"active","workspaces":1}`, prefix: "exec -T core"},
|
|
} {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
runner := &workspaceVerificationRunner{
|
|
running: test.running,
|
|
result: compose.Result{Stdout: test.payload},
|
|
}
|
|
if err := verifyRestoreWorkspace(context.Background(), installation, runner); err != nil {
|
|
t.Fatalf("verify restored workspace: %v", err)
|
|
}
|
|
if len(runner.calls) != 2 || !strings.Contains(runner.calls[1], test.prefix+" node /app/backend/dist/operator-command.js workspace-integrity") {
|
|
t.Fatalf("workspace verification calls = %#v", runner.calls)
|
|
}
|
|
for _, call := range runner.calls {
|
|
if strings.Contains(call, "curl") || strings.Contains(call, "-e const") || strings.Contains(strings.ToLower(call), "header") {
|
|
t.Fatalf("workspace verification used an unsafe command: %s", call)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestVerifyRestoreWorkspaceRejectsInvalidOperatorResults(t *testing.T) {
|
|
installation := preflightTestInstallation(t)
|
|
for _, test := range []struct {
|
|
name string
|
|
result compose.Result
|
|
err error
|
|
}{
|
|
{name: "empty"},
|
|
{name: "malformed", result: compose.Result{Stdout: `{malformed`}},
|
|
{name: "trailing document", result: compose.Result{Stdout: `{"ready":true,"state":"active","workspaces":1}{}`}},
|
|
{name: "unknown field", result: compose.Result{Stdout: `{"ready":true,"state":"active","workspaces":1,"detail":"unsafe"}`}},
|
|
{name: "not ready", result: compose.Result{Stdout: `{"ready":false,"state":"uninitialized","workspaces":0}`}},
|
|
{name: "unknown state", result: compose.Result{Stdout: `{"ready":true,"state":"unknown","workspaces":0}`}},
|
|
{name: "inconsistent count", result: compose.Result{Stdout: `{"ready":true,"state":"uninitialized","workspaces":1}`}},
|
|
{name: "nonzero", result: compose.Result{ExitCode: 2}, err: errors.New("exit status 2")},
|
|
} {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
runner := &workspaceVerificationRunner{result: test.result, err: test.err}
|
|
if err := verifyRestoreWorkspace(context.Background(), installation, runner); err == nil {
|
|
t.Fatal("invalid workspace verification result was accepted")
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func (lock fakeRestoreLock) Release() error {
|
|
if lock.release != nil {
|
|
lock.release()
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func equalStrings(got, want []string) bool {
|
|
if len(got) != len(want) {
|
|
return false
|
|
}
|
|
for index := range got {
|
|
if got[index] != want[index] {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
func restoreTestDependencies(t *testing.T, runner archiveRunner) restoreDependencies {
|
|
t.Helper()
|
|
return restoreDependencies{
|
|
preflight: func(ctx context.Context, installation config.Installation, request PreflightRequest) (PreflightResult, error) {
|
|
return Preflight(ctx, installation, request, permissivePreflightDependencies())
|
|
},
|
|
checkpoint: func(context.Context, config.Installation, CreateRequest) (Result, error) {
|
|
return Result{Path: "/tmp/default-checkpoint.zip"}, nil
|
|
},
|
|
prepareRecovery: func(context.Context, config.Installation, string) (PreflightResult, error) {
|
|
return PreflightResult{}, nil
|
|
},
|
|
recover: func(context.Context, config.Installation, PreflightResult, bool) error { return nil },
|
|
cleanupCheckpoint: func(string) error { return nil },
|
|
acquireLock: func(config.Installation) (restoreLock, error) { return fakeRestoreLock{}, nil },
|
|
runner: runner,
|
|
sleep: func(time.Duration) {},
|
|
restoreFile: func(context.Context, config.Installation, ArchiveEntryMetadata, io.Reader) error { return nil },
|
|
restoreVolume: func(context.Context, config.Installation, VolumeMetadata, io.Reader) error {
|
|
return nil
|
|
},
|
|
resetAuthenticationState: func(context.Context, config.Installation, archiveRunner) error {
|
|
return nil
|
|
},
|
|
verify: map[string]restoreVerify{
|
|
"health": func(context.Context, config.Installation, archiveRunner) error { return nil },
|
|
"doctor": func(context.Context, config.Installation, archiveRunner) error { return nil },
|
|
"pi": func(context.Context, config.Installation, archiveRunner) error { return nil },
|
|
"workspace": func(context.Context, config.Installation, archiveRunner) error { return nil },
|
|
},
|
|
}
|
|
}
|