feat(server): activate projected authentication safely
This commit is contained in:
@@ -175,6 +175,23 @@ func TestCreateReferencesAuthFilesByDefaultAndArchivesThemOnlyWithSecretCustody(
|
||||
}
|
||||
prepareBackupFixturePrivatePaths(t, []string{authDirectory}, []string{authPath, usersPath})
|
||||
fixture.installation.Authentication.ConfigDirectory = authDirectory
|
||||
runtimeRoot := filepath.Join(filepath.Dir(fixture.installation.Path), "auth-runtime")
|
||||
if err := os.Mkdir(runtimeRoot, 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runtimeSentinel := []byte("runtime-projection-must-not-be-archived")
|
||||
runtimeFiles := []string{
|
||||
filepath.Join(runtimeRoot, "CURRENT"), filepath.Join(runtimeRoot, "manifest.json"),
|
||||
filepath.Join(runtimeRoot, ".stage-test"), filepath.Join(runtimeRoot, ".current-test.tmp"),
|
||||
filepath.Join(runtimeRoot, ".auth-transaction.lock"), filepath.Join(runtimeRoot, ".auth.lock"),
|
||||
}
|
||||
for _, path := range runtimeFiles {
|
||||
if err := os.WriteFile(path, runtimeSentinel, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
prepareBackupFixturePrivatePaths(t, []string{runtimeRoot}, runtimeFiles)
|
||||
fixture.installation.Authentication.RuntimeProjection = &config.RuntimeProjection{Directory: runtimeRoot, UID: 10001, GID: 10001}
|
||||
|
||||
defaultOutput := filepath.Join(t.TempDir(), "default.zip")
|
||||
defaultResult, err := createWithDependencies(context.Background(), fixture.installation, CreateRequest{Output: defaultOutput}, testDependencies(t, newBackupRunner(fixture.installation, false)))
|
||||
@@ -204,6 +221,16 @@ func TestCreateReferencesAuthFilesByDefaultAndArchivesThemOnlyWithSecretCustody(
|
||||
t.Fatalf("secret backup warning = %q, want custody guidance", secretResult.Warning)
|
||||
}
|
||||
secretArchive := readFixtureArchive(t, secretOutput)
|
||||
for path, contents := range secretArchive.files {
|
||||
if bytes.Contains(contents, runtimeSentinel) {
|
||||
t.Fatalf("backup payload includes runtime projection data at %q", path)
|
||||
}
|
||||
}
|
||||
for _, entry := range secretArchive.manifest.Entries {
|
||||
if strings.HasPrefix(entry.SourcePath, runtimeRoot+string(filepath.Separator)) || strings.Contains(entry.Path, "auth-runtime") {
|
||||
t.Fatalf("backup manifest references runtime projection entry %#v", entry)
|
||||
}
|
||||
}
|
||||
for _, path := range []string{authPath, usersPath} {
|
||||
if !manifestHasArchivedSecret(secretArchive.manifest, path) {
|
||||
t.Fatalf("secret backup did not archive authentication file %q", path)
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"io"
|
||||
"time"
|
||||
|
||||
"github.com/aritmolab/thothii/tools/tht/internal/authconfig"
|
||||
"github.com/aritmolab/thothii/tools/tht/internal/config"
|
||||
"github.com/aritmolab/thothii/tools/tht/internal/lifecycle"
|
||||
)
|
||||
@@ -32,13 +33,20 @@ type RestoreResult struct {
|
||||
|
||||
type restoreVerify func(context.Context, config.Installation, archiveRunner) error
|
||||
|
||||
type authProjectionRestoreTransaction interface {
|
||||
PublishCanonical() (authconfig.ProjectionStatus, error)
|
||||
RestorePriorIfCanonicalUnchanged() error
|
||||
Close() error
|
||||
}
|
||||
|
||||
type restoreDependencies struct {
|
||||
preflight func(context.Context, config.Installation, PreflightRequest) (PreflightResult, error)
|
||||
// checkpoint requires the opaque capability created by lifecycle acquisition. It must not call
|
||||
// public Create, which would re-acquire the non-reentrant lock and deadlock the transaction.
|
||||
checkpoint func(context.Context, *lifecycle.Transaction, config.Installation, CreateRequest) (Result, error)
|
||||
prepareRecovery func(context.Context, config.Installation, string) (PreflightResult, error)
|
||||
recover func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool) error
|
||||
recover func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool, authProjectionRestoreTransaction) error
|
||||
beginAuthProjection func(context.Context, config.Installation) (authProjectionRestoreTransaction, error)
|
||||
cleanupCheckpoint func(string) error
|
||||
acquireTransaction func(config.Installation) (*lifecycle.Transaction, error)
|
||||
runner archiveRunner
|
||||
@@ -107,6 +115,12 @@ func restoreWithDependencies(ctx context.Context, installation config.Installati
|
||||
return RestoreResult{}, err
|
||||
}
|
||||
defer preflight.CloseArchive()
|
||||
authRestoreRequired := installation.HasRuntimeAuthProjection() && manifestArchivesAuthentication(preflight.Manifest)
|
||||
if authRestoreRequired {
|
||||
if err := requireAuthProjectionRestorePrivilege(); err != nil {
|
||||
return result, err
|
||||
}
|
||||
}
|
||||
|
||||
checkpoint, err := deps.checkpoint(ctx, transaction, installation, CreateRequest{IncludeSecrets: true, Confirm: true})
|
||||
if err != nil {
|
||||
@@ -150,11 +164,24 @@ func restoreWithDependencies(ctx context.Context, installation config.Installati
|
||||
return result, errors.Join(err, cleanupErr)
|
||||
}
|
||||
|
||||
var authTransaction authProjectionRestoreTransaction
|
||||
if authRestoreRequired {
|
||||
if deps.beginAuthProjection == nil {
|
||||
cleanupErr := deps.cleanupCheckpoint(checkpoint.Path)
|
||||
return result, errors.Join(errors.New("restore authentication projection dependency is unavailable"), cleanupErr)
|
||||
}
|
||||
authTransaction, err = deps.beginAuthProjection(ctx, installation)
|
||||
if err != nil {
|
||||
cleanupErr := deps.cleanupCheckpoint(checkpoint.Path)
|
||||
return result, errors.Join(errors.New("restore authentication projection could not be blocked"), cleanupErr)
|
||||
}
|
||||
}
|
||||
|
||||
state := restoreTransactionState{}
|
||||
defer func() {
|
||||
if state.recoveryRequired(resultErr) {
|
||||
recoveryContext, cancel := boundedCleanupContext()
|
||||
recoveryErr := deps.recover(recoveryContext, installation, recovery, recoveryStage, state.wasRunning)
|
||||
recoveryErr := deps.recover(recoveryContext, installation, recovery, recoveryStage, state.wasRunning, authTransaction)
|
||||
cancel()
|
||||
if recoveryErr != nil {
|
||||
resultErr = errors.Join(resultErr, fmt.Errorf("restore recovery checkpoint: %w", recoveryErr))
|
||||
@@ -174,6 +201,20 @@ func restoreWithDependencies(ctx context.Context, installation config.Installati
|
||||
checkpointCleanupSucceeded = false
|
||||
cleanupErr = errors.Join(cleanupErr, fmt.Errorf("destroy recovery checkpoint: %w", checkpointErr))
|
||||
}
|
||||
authCleanupSucceeded := true
|
||||
if authTransaction != nil {
|
||||
if resultErr != nil && !state.mutated {
|
||||
if restoreErr := authTransaction.RestorePriorIfCanonicalUnchanged(); restoreErr != nil {
|
||||
authCleanupSucceeded = false
|
||||
cleanupErr = errors.Join(cleanupErr, errors.New("restore authentication projection could not restore its prior selector"))
|
||||
}
|
||||
}
|
||||
if closeErr := authTransaction.Close(); closeErr != nil {
|
||||
authCleanupSucceeded = false
|
||||
cleanupErr = errors.Join(cleanupErr, errors.New("restore authentication projection transaction could not be closed"))
|
||||
}
|
||||
authTransaction = nil
|
||||
}
|
||||
if !state.maintenanceAttempted {
|
||||
if cleanupErr != nil {
|
||||
result = RestoreResult{}
|
||||
@@ -204,7 +245,7 @@ func restoreWithDependencies(ctx context.Context, installation config.Installati
|
||||
}
|
||||
// A failed checkpoint recovery deliberately leaves admissions blocked. Starting or
|
||||
// deactivating at that point would expose an unverified, possibly partial restore.
|
||||
if state.mayDeactivateMaintenance() && restartCompleted {
|
||||
if state.mayDeactivateMaintenance() && restartCompleted && authCleanupSucceeded {
|
||||
deactivateErr, deactivated := retryBoundedCleanup(func(cleanupContext context.Context) error {
|
||||
return maintenance(cleanupContext, installation, deps.runner, false)
|
||||
})
|
||||
@@ -250,6 +291,11 @@ func restoreWithDependencies(ctx context.Context, installation config.Installati
|
||||
if err := deps.resetAuthenticationState(ctx, installation, deps.runner); err != nil {
|
||||
return result, fmt.Errorf("reset authentication state: %w", err)
|
||||
}
|
||||
if authTransaction != nil {
|
||||
if err := publishRestoredAuthentication(authTransaction); err != nil {
|
||||
return result, err
|
||||
}
|
||||
}
|
||||
if state.wasRunning {
|
||||
if err := composeStartAndVerify(ctx, installation, deps.runner); err != nil {
|
||||
return result, err
|
||||
@@ -265,6 +311,26 @@ func restoreWithDependencies(ctx context.Context, installation config.Installati
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func manifestArchivesAuthentication(manifest Manifest) bool {
|
||||
for _, entry := range manifest.Entries {
|
||||
if entry.Archived && entry.Kind == EntryExternalSecret && entry.Owner == "authentication-configuration" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func publishRestoredAuthentication(transaction authProjectionRestoreTransaction) error {
|
||||
status, err := transaction.PublishCanonical()
|
||||
if err != nil || status.State != "ready" || !status.Equal {
|
||||
if err != nil {
|
||||
return fmt.Errorf("publish restored authentication projection: %w", err)
|
||||
}
|
||||
return errors.New("publish restored authentication projection")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func ensureCombinedRestoreCapacity(candidate, recovery PreflightResult) error {
|
||||
if candidate.freeBytes == nil || candidate.stagingRoot == "" || candidate.stagingRoot != recovery.stagingRoot {
|
||||
return errors.New("candidate and recovery archives do not share controlled restore staging")
|
||||
|
||||
@@ -51,9 +51,20 @@ func productionRestoreDependencies(installation config.Installation) restoreDepe
|
||||
},
|
||||
cleanupCheckpoint: cleanupRecoveryCheckpoint,
|
||||
acquireTransaction: lifecycle.AcquireTransaction,
|
||||
runner: runner,
|
||||
sleep: time.Sleep,
|
||||
restoreFile: restoreFilePayload,
|
||||
beginAuthProjection: func(ctx context.Context, target config.Installation) (authProjectionRestoreTransaction, error) {
|
||||
projection := target.RuntimeAuthProjection()
|
||||
if projection == nil {
|
||||
return nil, errors.New("runtime authentication projection is unavailable")
|
||||
}
|
||||
return authconfig.BeginExternalProjectionTransaction(ctx, target.AuthenticationDirectory(), authconfig.ProjectionSpec{
|
||||
RuntimeRoot: projection.Directory,
|
||||
UID: projection.UID,
|
||||
GID: projection.GID,
|
||||
})
|
||||
},
|
||||
runner: runner,
|
||||
sleep: time.Sleep,
|
||||
restoreFile: restoreFilePayload,
|
||||
restoreVolume: func(ctx context.Context, _ config.Installation, volume VolumeMetadata, input io.Reader) error {
|
||||
result, err := runner.Stream(ctx, volumeRestoreCommand(volume.Name), input, io.Discard)
|
||||
if err != nil || result.ExitCode != 0 {
|
||||
@@ -75,8 +86,8 @@ func productionRestoreDependencies(installation config.Installation) restoreDepe
|
||||
deps.prepareRecovery = func(ctx context.Context, target config.Installation, path string) (PreflightResult, error) {
|
||||
return deps.preflight(ctx, target, PreflightRequest{Archive: path, Confirm: true, AllowExternalSecrets: true})
|
||||
}
|
||||
deps.recover = func(ctx context.Context, target config.Installation, recovery PreflightResult, staged *stagedArchive, wasRunning bool) error {
|
||||
return recoverRestoreTransaction(ctx, target, recovery, staged, wasRunning, deps)
|
||||
deps.recover = func(ctx context.Context, target config.Installation, recovery PreflightResult, staged *stagedArchive, wasRunning bool, transaction authProjectionRestoreTransaction) error {
|
||||
return recoverRestoreTransaction(ctx, target, recovery, staged, wasRunning, deps, transaction)
|
||||
}
|
||||
return deps
|
||||
}
|
||||
@@ -88,7 +99,7 @@ func cleanupRecoveryCheckpoint(path string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func recoverRestoreTransaction(ctx context.Context, installation config.Installation, recovery PreflightResult, staged *stagedArchive, wasRunning bool, deps restoreDependencies) (resultErr error) {
|
||||
func recoverRestoreTransaction(ctx context.Context, installation config.Installation, recovery PreflightResult, staged *stagedArchive, wasRunning bool, deps restoreDependencies, transaction authProjectionRestoreTransaction) (resultErr error) {
|
||||
if staged == nil || staged.file == nil {
|
||||
return errors.New("recovery checkpoint was not staged before restore mutation")
|
||||
}
|
||||
@@ -109,6 +120,11 @@ func recoverRestoreTransaction(ctx context.Context, installation config.Installa
|
||||
if err := deps.resetAuthenticationState(ctx, installation, deps.runner); err != nil {
|
||||
return errors.Join(resultErr, err)
|
||||
}
|
||||
if transaction != nil {
|
||||
if err := publishRestoredAuthentication(transaction); err != nil {
|
||||
return errors.Join(resultErr, err)
|
||||
}
|
||||
}
|
||||
if wasRunning {
|
||||
if err := composeStartAndVerify(ctx, installation, deps.runner); err != nil {
|
||||
resultErr = errors.Join(resultErr, err)
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
//go:build linux
|
||||
|
||||
package backup
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
)
|
||||
|
||||
var restoreProjectionEffectiveUID = os.Geteuid
|
||||
|
||||
func requireAuthProjectionRestorePrivilege() error {
|
||||
if restoreProjectionEffectiveUID() != 0 {
|
||||
return errors.New("projected authentication restore requires root")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
//go:build linux
|
||||
|
||||
package backup
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
"github.com/aritmolab/thothii/tools/tht/internal/config"
|
||||
"github.com/aritmolab/thothii/tools/tht/internal/lifecycle"
|
||||
)
|
||||
|
||||
func TestRestoreAuthBearingArchiveRefusesNonRootBeforeTransactionOrWrite(t *testing.T) {
|
||||
installation, archive := projectedRestoreFixture(t)
|
||||
deps := restoreTestDependencies(t, newBackupRunner(installation, false))
|
||||
checkpointCalled := false
|
||||
beginCalled := false
|
||||
writeCalled := false
|
||||
deps.checkpoint = func(context.Context, *lifecycle.Transaction, config.Installation, CreateRequest) (Result, error) {
|
||||
checkpointCalled = true
|
||||
return Result{}, nil
|
||||
}
|
||||
deps.beginAuthProjection = func(context.Context, config.Installation) (authProjectionRestoreTransaction, error) {
|
||||
beginCalled = true
|
||||
return nil, nil
|
||||
}
|
||||
deps.restoreFile = func(context.Context, config.Installation, ArchiveEntryMetadata, io.Reader) error {
|
||||
writeCalled = true
|
||||
return nil
|
||||
}
|
||||
previous := restoreProjectionEffectiveUID
|
||||
restoreProjectionEffectiveUID = func() int { return 1000 }
|
||||
t.Cleanup(func() { restoreProjectionEffectiveUID = previous })
|
||||
|
||||
if _, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: archive, Confirm: true}, deps); err == nil {
|
||||
t.Fatal("projected authentication restore unexpectedly accepted non-root execution")
|
||||
}
|
||||
if checkpointCalled || beginCalled || writeCalled {
|
||||
t.Fatalf("non-root restore crossed mutation boundary: checkpoint=%t begin=%t write=%t", checkpointCalled, beginCalled, writeCalled)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
//go:build !linux
|
||||
|
||||
package backup
|
||||
|
||||
import "errors"
|
||||
|
||||
func requireAuthProjectionRestorePrivilege() error {
|
||||
return errors.New("projected authentication restore is unsupported")
|
||||
}
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aritmolab/thothii/tools/tht/internal/authconfig"
|
||||
"github.com/aritmolab/thothii/tools/tht/internal/compose"
|
||||
"github.com/aritmolab/thothii/tools/tht/internal/config"
|
||||
"github.com/aritmolab/thothii/tools/tht/internal/lifecycle"
|
||||
@@ -624,7 +625,7 @@ func TestRestoreLifecycleLockExcludesCompetingTransactionsUntilTerminalCleanup(t
|
||||
}
|
||||
return targetFailure
|
||||
}
|
||||
deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool) error {
|
||||
deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool, authProjectionRestoreTransaction) error {
|
||||
if err := gate("recovery"); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -645,7 +646,7 @@ func TestRestoreLifecycleLockExcludesCompetingTransactionsUntilTerminalCleanup(t
|
||||
}
|
||||
return targetFailure
|
||||
}
|
||||
deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool) error {
|
||||
deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool, authProjectionRestoreTransaction) error {
|
||||
if err := gate("recovery-failure"); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -666,7 +667,7 @@ func TestRestoreLifecycleLockExcludesCompetingTransactionsUntilTerminalCleanup(t
|
||||
cancel()
|
||||
return context.Canceled
|
||||
}
|
||||
deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool) error {
|
||||
deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool, authProjectionRestoreTransaction) error {
|
||||
if err := gate("recovery"); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -892,7 +893,7 @@ func TestRestoreCannotApplyAStaleCheckpointOverAnInterleavedRestore(t *testing.T
|
||||
firstDeps.restoreFile = func(context.Context, config.Installation, ArchiveEntryMetadata, io.Reader) error {
|
||||
return errors.New("first target mutation failed before changing state")
|
||||
}
|
||||
firstDeps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool) error {
|
||||
firstDeps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool, authProjectionRestoreTransaction) error {
|
||||
recoveryObserved = targetState
|
||||
targetState = checkpointState
|
||||
firstRunner.running, firstRunner.coreRunning = true, true
|
||||
@@ -1095,7 +1096,7 @@ func TestRestoreFileFailureRollsBackSecretAwareCheckpointBeforeCleanup(t *testin
|
||||
return Result{Path: "/tmp/recovery.zip"}, nil
|
||||
}
|
||||
var events []string
|
||||
deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool) error {
|
||||
deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool, authProjectionRestoreTransaction) error {
|
||||
events = append(events, "recover")
|
||||
return nil
|
||||
}
|
||||
@@ -1135,7 +1136,7 @@ func TestRestoreFailureAfterAuthenticationMutationRollsBackAndClearsRuntimeState
|
||||
events = append(events, "auth-runtime-reset-failed")
|
||||
return resetErr
|
||||
}
|
||||
deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool) error {
|
||||
deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool, authProjectionRestoreTransaction) error {
|
||||
events = append(events, "secret-aware-recovery-and-reauth-reset")
|
||||
return nil
|
||||
}
|
||||
@@ -1161,7 +1162,7 @@ func TestRestoreCleanupFailureDoesNotSuppressRollback(t *testing.T) {
|
||||
cleanupErr := errors.New("checkpoint cleanup failure")
|
||||
recovered := false
|
||||
deps.restoreFile = func(context.Context, config.Installation, ArchiveEntryMetadata, io.Reader) error { return mutationErr }
|
||||
deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool) error {
|
||||
deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool, authProjectionRestoreTransaction) error {
|
||||
recovered = true
|
||||
return nil
|
||||
}
|
||||
@@ -1223,7 +1224,7 @@ func TestRestoreStartFailureRecoversPreviouslyRunningTarget(t *testing.T) {
|
||||
backingRunner := newBackupRunner(installation, true)
|
||||
deps := restoreTestDependencies(t, failStartRestoreRunner{fakeBackupRunner: backingRunner})
|
||||
recovered := false
|
||||
deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool) error {
|
||||
deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool, authProjectionRestoreTransaction) error {
|
||||
recovered = true
|
||||
backingRunner.running = true
|
||||
return nil
|
||||
@@ -1246,7 +1247,7 @@ func TestRestoreVerificationFailureRecoversPreviouslyRunningTarget(t *testing.T)
|
||||
verificationErr := errors.New("Pi is unavailable")
|
||||
deps.verify["pi"] = func(context.Context, config.Installation, archiveRunner) error { return verificationErr }
|
||||
recovered := false
|
||||
deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool) error {
|
||||
deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool, authProjectionRestoreTransaction) error {
|
||||
recovered = true
|
||||
return nil
|
||||
}
|
||||
@@ -1352,7 +1353,7 @@ func TestRestoreRecoversBehindBarrierForEveryVerificationFailure(t *testing.T) {
|
||||
}
|
||||
var recoveryBarrierActive bool
|
||||
var recoveryContext cleanupContextObservation
|
||||
deps.recover = func(ctx context.Context, _ config.Installation, _ PreflightResult, _ *stagedArchive, _ bool) error {
|
||||
deps.recover = func(ctx context.Context, _ config.Installation, _ PreflightResult, _ *stagedArchive, _ bool, _ authProjectionRestoreTransaction) error {
|
||||
recoveryContext = observeCleanupContext(ctx)
|
||||
recoveryBarrierActive = runner.maintenance
|
||||
runner.running, runner.coreRunning = true, true
|
||||
@@ -1392,7 +1393,7 @@ func TestRestoreDoesNotRollbackAfterFinalDeactivationResponseLoss(t *testing.T)
|
||||
}
|
||||
deps := restoreTestDependencies(t, runner)
|
||||
recoveryCalls := 0
|
||||
deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool) error {
|
||||
deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool, authProjectionRestoreTransaction) error {
|
||||
recoveryCalls++
|
||||
return nil
|
||||
}
|
||||
@@ -1450,7 +1451,7 @@ func TestRestoreUsesBoundedRecoveryContextAfterPostMutationCancellation(t *testi
|
||||
}
|
||||
var recoveryContext cleanupContextObservation
|
||||
var recoveryBarrierActive bool
|
||||
deps.recover = func(ctx context.Context, _ config.Installation, _ PreflightResult, _ *stagedArchive, _ bool) error {
|
||||
deps.recover = func(ctx context.Context, _ config.Installation, _ PreflightResult, _ *stagedArchive, _ bool, _ authProjectionRestoreTransaction) error {
|
||||
recoveryContext = observeCleanupContext(ctx)
|
||||
recoveryBarrierActive = runner.maintenance
|
||||
runner.running, runner.coreRunning = true, true
|
||||
@@ -1536,7 +1537,7 @@ func TestRecoverRestoreTransactionVerifiesRecoveredStateBeforeReturning(t *testi
|
||||
}
|
||||
|
||||
staged := stageRecoveryForTest(t, installation, recovery)
|
||||
if err := recoverRestoreTransaction(context.Background(), installation, recovery, staged, true, deps); err != nil {
|
||||
if err := recoverRestoreTransaction(context.Background(), installation, recovery, staged, true, deps, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, want := checks, []string{"health", "doctor", "pi", "workspace"}; !equalStrings(got, want) {
|
||||
@@ -1566,7 +1567,7 @@ func TestRecoverRestoreTransactionFailsClosedForEveryVerification(t *testing.T)
|
||||
}
|
||||
|
||||
staged := stageRecoveryForTest(t, installation, recovery)
|
||||
err = recoverRestoreTransaction(context.Background(), installation, recovery, staged, true, deps)
|
||||
err = recoverRestoreTransaction(context.Background(), installation, recovery, staged, true, deps, nil)
|
||||
if !errors.Is(err, verificationErr) {
|
||||
t.Fatalf("recoverRestoreTransaction() error = %v, want %v", err, verificationErr)
|
||||
}
|
||||
@@ -1646,8 +1647,8 @@ func TestRestoreReleasesBarrierOnlyAfterVerifiedRecoveryFromLostResponse(t *test
|
||||
}
|
||||
return nil
|
||||
}
|
||||
deps.recover = func(ctx context.Context, target config.Installation, checkpoint PreflightResult, staged *stagedArchive, wasRunning bool) error {
|
||||
return recoverRestoreTransaction(ctx, target, checkpoint, staged, wasRunning, deps)
|
||||
deps.recover = func(ctx context.Context, target config.Installation, checkpoint PreflightResult, staged *stagedArchive, wasRunning bool, transaction authProjectionRestoreTransaction) error {
|
||||
return recoverRestoreTransaction(ctx, target, checkpoint, staged, wasRunning, deps, transaction)
|
||||
}
|
||||
|
||||
_, err = restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: archive, Confirm: true}, deps)
|
||||
@@ -1744,7 +1745,7 @@ func TestRestoreCleansMaintenanceAfterMutationAndRollbackFailures(t *testing.T)
|
||||
deps.restoreFile = func(context.Context, config.Installation, ArchiveEntryMetadata, io.Reader) error {
|
||||
return mutationErr
|
||||
}
|
||||
deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool) error {
|
||||
deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool, authProjectionRestoreTransaction) error {
|
||||
if test.recoveryErr == nil {
|
||||
backing.running, backing.coreRunning = true, true
|
||||
}
|
||||
@@ -2005,7 +2006,9 @@ func restoreTestDependencies(t *testing.T, runner archiveRunner) restoreDependen
|
||||
prepareRecovery: func(ctx context.Context, installation config.Installation, _ string) (PreflightResult, error) {
|
||||
return Preflight(ctx, installation, PreflightRequest{Archive: recoveryArchive, Confirm: true, AllowExternalSecrets: true}, permissivePreflightDependencies())
|
||||
},
|
||||
recover: func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool) error { return nil },
|
||||
recover: func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool, authProjectionRestoreTransaction) error {
|
||||
return nil
|
||||
},
|
||||
cleanupCheckpoint: func(string) error { return nil },
|
||||
acquireTransaction: lifecycle.AcquireTransaction,
|
||||
runner: runner,
|
||||
@@ -2025,3 +2028,235 @@ func restoreTestDependencies(t *testing.T, runner archiveRunner) restoreDependen
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
type projectionRestoreStub struct {
|
||||
events *[]string
|
||||
blocked bool
|
||||
publishErr error
|
||||
restoreErr error
|
||||
closeErr error
|
||||
}
|
||||
|
||||
func (stub *projectionRestoreStub) PublishCanonical() (authconfig.ProjectionStatus, error) {
|
||||
*stub.events = append(*stub.events, "publish")
|
||||
if stub.publishErr != nil {
|
||||
return authconfig.ProjectionStatus{}, stub.publishErr
|
||||
}
|
||||
stub.blocked = false
|
||||
return authconfig.ProjectionStatus{State: "ready", Generation: "g", CanonicalRevision: "sha256:g", Equal: true}, nil
|
||||
}
|
||||
|
||||
func (stub *projectionRestoreStub) RestorePriorIfCanonicalUnchanged() error {
|
||||
*stub.events = append(*stub.events, "restore-prior")
|
||||
if stub.restoreErr != nil {
|
||||
return stub.restoreErr
|
||||
}
|
||||
stub.blocked = false
|
||||
return nil
|
||||
}
|
||||
|
||||
func (stub *projectionRestoreStub) Close() error {
|
||||
*stub.events = append(*stub.events, "close")
|
||||
return stub.closeErr
|
||||
}
|
||||
|
||||
type restartEventRunner struct {
|
||||
archiveRunner
|
||||
events *[]string
|
||||
}
|
||||
|
||||
func (runner restartEventRunner) Run(ctx context.Context, args []string, input io.Reader) (compose.Result, error) {
|
||||
if strings.HasSuffix(strings.Join(args, " "), " start") {
|
||||
*runner.events = append(*runner.events, "restart")
|
||||
}
|
||||
return runner.archiveRunner.Run(ctx, args, input)
|
||||
}
|
||||
|
||||
func (runner restartEventRunner) Stream(ctx context.Context, args []string, input io.Reader, output io.Writer) (compose.Result, error) {
|
||||
return runner.archiveRunner.Stream(ctx, args, input, output)
|
||||
}
|
||||
|
||||
func (runner restartEventRunner) SessionInventoryScope() string {
|
||||
return runner.archiveRunner.SessionInventoryScope()
|
||||
}
|
||||
|
||||
func projectedRestoreFixture(t *testing.T) (config.Installation, string) {
|
||||
t.Helper()
|
||||
installation := preflightTestInstallation(t)
|
||||
authRoot := filepath.Join(t.TempDir(), "canonical-auth")
|
||||
if err := os.Mkdir(authRoot, 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
installation.Authentication.ConfigDirectory = authRoot
|
||||
installation.Authentication.RuntimeProjection = &config.RuntimeProjection{Directory: filepath.Join(t.TempDir(), "runtime-auth"), UID: 10001, GID: 10001}
|
||||
archive := filepath.Join(t.TempDir(), "auth-restore.zip")
|
||||
writePreflightArchive(t, archive, preflightArchiveSpec{includeSecrets: true, entries: []preflightArchiveEntry{{
|
||||
path: "authentication-secrets/000-auth.yaml", body: []byte("candidate auth\n"), kind: EntryExternalSecret,
|
||||
sensitive: true, owner: "authentication-configuration", sourcePath: filepath.Join(authRoot, "auth.yaml"),
|
||||
}}})
|
||||
return installation, archive
|
||||
}
|
||||
|
||||
func assertOrderedEvents(t *testing.T, events []string, wants ...string) {
|
||||
t.Helper()
|
||||
at := 0
|
||||
for _, want := range wants {
|
||||
for at < len(events) && events[at] != want {
|
||||
at++
|
||||
}
|
||||
if at == len(events) {
|
||||
t.Fatalf("events = %v, want ordered subsequence %v", events, wants)
|
||||
}
|
||||
at++
|
||||
}
|
||||
}
|
||||
|
||||
func TestRestoreAuthBearingArchiveBlocksBeforeWritePublishesBeforeRestart(t *testing.T) {
|
||||
installation, archive := projectedRestoreFixture(t)
|
||||
var events []string
|
||||
backing := newBackupRunner(installation, true)
|
||||
runner := restartEventRunner{archiveRunner: backing, events: &events}
|
||||
deps := restoreTestDependencies(t, runner)
|
||||
transaction := &projectionRestoreStub{events: &events}
|
||||
deps.beginAuthProjection = func(context.Context, config.Installation) (authProjectionRestoreTransaction, error) {
|
||||
transaction.blocked = true
|
||||
events = append(events, "begin")
|
||||
return transaction, nil
|
||||
}
|
||||
deps.restoreFile = func(_ context.Context, _ config.Installation, entry ArchiveEntryMetadata, _ io.Reader) error {
|
||||
if entry.Owner == "authentication-configuration" {
|
||||
if !transaction.blocked {
|
||||
t.Fatal("auth destination write began without blocked projection")
|
||||
}
|
||||
events = append(events, "restore-auth")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if _, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: archive, Confirm: true}, deps); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
assertOrderedEvents(t, events, "begin", "restore-auth", "publish", "restart", "close")
|
||||
if transaction.blocked {
|
||||
t.Fatalf("successful restore closed while projection remained blocked: %v", events)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRestoreAuthCandidatePublicationFailureRecoversThenStaysBlockedWithoutRestart(t *testing.T) {
|
||||
installation, archive := projectedRestoreFixture(t)
|
||||
var events []string
|
||||
backing := newBackupRunner(installation, true)
|
||||
runner := restartEventRunner{archiveRunner: backing, events: &events}
|
||||
deps := restoreTestDependencies(t, runner)
|
||||
publicationErr := errors.New("synthetic publication failure")
|
||||
transaction := &projectionRestoreStub{events: &events, publishErr: publicationErr}
|
||||
deps.beginAuthProjection = func(context.Context, config.Installation) (authProjectionRestoreTransaction, error) {
|
||||
transaction.blocked = true
|
||||
return transaction, nil
|
||||
}
|
||||
deps.recover = func(_ context.Context, _ config.Installation, _ PreflightResult, _ *stagedArchive, _ bool, transaction authProjectionRestoreTransaction) error {
|
||||
events = append(events, "restore-checkpoint")
|
||||
_, err := transaction.PublishCanonical()
|
||||
return err
|
||||
}
|
||||
if _, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: archive, Confirm: true}, deps); !errors.Is(err, publicationErr) {
|
||||
t.Fatalf("restore error = %v, want publication failure", err)
|
||||
}
|
||||
assertOrderedEvents(t, events, "publish", "restore-checkpoint", "close")
|
||||
if !transaction.blocked {
|
||||
t.Fatal("publication failure reopened projection")
|
||||
}
|
||||
if backing.startCount != 0 {
|
||||
t.Fatalf("restart after failed candidate/recovery publication = %d", backing.startCount)
|
||||
}
|
||||
if !backing.maintenance {
|
||||
t.Fatal("admissions reopened after failed recovery publication")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRestoreAuthVerificationFailuresRepublishCheckpointBeforeRecoveryRestart(t *testing.T) {
|
||||
for _, failed := range []string{"health", "doctor", "pi", "workspace"} {
|
||||
t.Run(failed, func(t *testing.T) {
|
||||
installation, archive := projectedRestoreFixture(t)
|
||||
var events []string
|
||||
backing := newBackupRunner(installation, true)
|
||||
runner := restartEventRunner{archiveRunner: backing, events: &events}
|
||||
deps := restoreTestDependencies(t, runner)
|
||||
transaction := &projectionRestoreStub{events: &events}
|
||||
deps.beginAuthProjection = func(context.Context, config.Installation) (authProjectionRestoreTransaction, error) {
|
||||
transaction.blocked = true
|
||||
return transaction, nil
|
||||
}
|
||||
for _, name := range []string{"health", "doctor", "pi", "workspace"} {
|
||||
name := name
|
||||
deps.verify[name] = func(context.Context, config.Installation, archiveRunner) error {
|
||||
events = append(events, "candidate-"+name)
|
||||
if name == failed {
|
||||
return errors.New("synthetic " + name + " failure")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
deps.recover = func(ctx context.Context, target config.Installation, _ PreflightResult, _ *stagedArchive, wasRunning bool, transaction authProjectionRestoreTransaction) error {
|
||||
events = append(events, "restore-checkpoint")
|
||||
if _, err := transaction.PublishCanonical(); err != nil {
|
||||
return err
|
||||
}
|
||||
events = append(events, "verify-checkpoint")
|
||||
if wasRunning {
|
||||
return composeStartAndVerify(ctx, target, runner)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if _, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: archive, Confirm: true}, deps); err == nil {
|
||||
t.Fatalf("%s failure accepted", failed)
|
||||
}
|
||||
assertOrderedEvents(t, events, "publish", "candidate-"+failed, "restore-checkpoint", "publish", "verify-checkpoint", "restart", "close")
|
||||
if transaction.blocked {
|
||||
t.Fatalf("verified checkpoint recovery remained blocked: %v", events)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRestoreAuthPreMutationFailureRestoresPriorReadySelector(t *testing.T) {
|
||||
installation, archive := projectedRestoreFixture(t)
|
||||
var events []string
|
||||
backing := newBackupRunner(installation, false)
|
||||
runner := &commandFailureRunner{fakeBackupRunner: backing, failures: []*commandFailure{{
|
||||
match: func(command string) bool { return strings.Contains(command, " ps --all --format json") },
|
||||
err: errors.New("synthetic pre-mutation failure"), remaining: 1,
|
||||
}}}
|
||||
deps := restoreTestDependencies(t, runner)
|
||||
transaction := &projectionRestoreStub{events: &events}
|
||||
deps.beginAuthProjection = func(context.Context, config.Installation) (authProjectionRestoreTransaction, error) {
|
||||
transaction.blocked = true
|
||||
return transaction, nil
|
||||
}
|
||||
deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool, authProjectionRestoreTransaction) error {
|
||||
t.Fatal("recovery ran before any destination mutation")
|
||||
return nil
|
||||
}
|
||||
if _, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: archive, Confirm: true}, deps); err == nil {
|
||||
t.Fatal("pre-mutation failure accepted")
|
||||
}
|
||||
assertOrderedEvents(t, events, "restore-prior", "close")
|
||||
if transaction.blocked {
|
||||
t.Fatal("unchanged canonical pre-write failure did not restore ready selector")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRestoreNonAuthArchiveNeverBeginsProjection(t *testing.T) {
|
||||
installation := preflightTestInstallation(t)
|
||||
deps := restoreTestDependencies(t, newBackupRunner(installation, false))
|
||||
called := false
|
||||
deps.beginAuthProjection = func(context.Context, config.Installation) (authProjectionRestoreTransaction, error) {
|
||||
called = true
|
||||
return nil, errors.New("must not begin")
|
||||
}
|
||||
if _, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: restoreArchive(t), Confirm: true}, deps); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if called {
|
||||
t.Fatal("non-auth archive began projection transaction")
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user