feat(server): activate projected authentication safely

This commit is contained in:
User
2026-08-22 01:01:36 +02:00
parent 903c0b4de5
commit 3d9a9f0675
32 changed files with 1837 additions and 38 deletions
+27
View File
@@ -175,6 +175,23 @@ func TestCreateReferencesAuthFilesByDefaultAndArchivesThemOnlyWithSecretCustody(
}
prepareBackupFixturePrivatePaths(t, []string{authDirectory}, []string{authPath, usersPath})
fixture.installation.Authentication.ConfigDirectory = authDirectory
runtimeRoot := filepath.Join(filepath.Dir(fixture.installation.Path), "auth-runtime")
if err := os.Mkdir(runtimeRoot, 0o700); err != nil {
t.Fatal(err)
}
runtimeSentinel := []byte("runtime-projection-must-not-be-archived")
runtimeFiles := []string{
filepath.Join(runtimeRoot, "CURRENT"), filepath.Join(runtimeRoot, "manifest.json"),
filepath.Join(runtimeRoot, ".stage-test"), filepath.Join(runtimeRoot, ".current-test.tmp"),
filepath.Join(runtimeRoot, ".auth-transaction.lock"), filepath.Join(runtimeRoot, ".auth.lock"),
}
for _, path := range runtimeFiles {
if err := os.WriteFile(path, runtimeSentinel, 0o600); err != nil {
t.Fatal(err)
}
}
prepareBackupFixturePrivatePaths(t, []string{runtimeRoot}, runtimeFiles)
fixture.installation.Authentication.RuntimeProjection = &config.RuntimeProjection{Directory: runtimeRoot, UID: 10001, GID: 10001}
defaultOutput := filepath.Join(t.TempDir(), "default.zip")
defaultResult, err := createWithDependencies(context.Background(), fixture.installation, CreateRequest{Output: defaultOutput}, testDependencies(t, newBackupRunner(fixture.installation, false)))
@@ -204,6 +221,16 @@ func TestCreateReferencesAuthFilesByDefaultAndArchivesThemOnlyWithSecretCustody(
t.Fatalf("secret backup warning = %q, want custody guidance", secretResult.Warning)
}
secretArchive := readFixtureArchive(t, secretOutput)
for path, contents := range secretArchive.files {
if bytes.Contains(contents, runtimeSentinel) {
t.Fatalf("backup payload includes runtime projection data at %q", path)
}
}
for _, entry := range secretArchive.manifest.Entries {
if strings.HasPrefix(entry.SourcePath, runtimeRoot+string(filepath.Separator)) || strings.Contains(entry.Path, "auth-runtime") {
t.Fatalf("backup manifest references runtime projection entry %#v", entry)
}
}
for _, path := range []string{authPath, usersPath} {
if !manifestHasArchivedSecret(secretArchive.manifest, path) {
t.Fatalf("secret backup did not archive authentication file %q", path)
+69 -3
View File
@@ -8,6 +8,7 @@ import (
"io"
"time"
"github.com/aritmolab/thothii/tools/tht/internal/authconfig"
"github.com/aritmolab/thothii/tools/tht/internal/config"
"github.com/aritmolab/thothii/tools/tht/internal/lifecycle"
)
@@ -32,13 +33,20 @@ type RestoreResult struct {
type restoreVerify func(context.Context, config.Installation, archiveRunner) error
type authProjectionRestoreTransaction interface {
PublishCanonical() (authconfig.ProjectionStatus, error)
RestorePriorIfCanonicalUnchanged() error
Close() error
}
type restoreDependencies struct {
preflight func(context.Context, config.Installation, PreflightRequest) (PreflightResult, error)
// checkpoint requires the opaque capability created by lifecycle acquisition. It must not call
// public Create, which would re-acquire the non-reentrant lock and deadlock the transaction.
checkpoint func(context.Context, *lifecycle.Transaction, config.Installation, CreateRequest) (Result, error)
prepareRecovery func(context.Context, config.Installation, string) (PreflightResult, error)
recover func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool) error
recover func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool, authProjectionRestoreTransaction) error
beginAuthProjection func(context.Context, config.Installation) (authProjectionRestoreTransaction, error)
cleanupCheckpoint func(string) error
acquireTransaction func(config.Installation) (*lifecycle.Transaction, error)
runner archiveRunner
@@ -107,6 +115,12 @@ func restoreWithDependencies(ctx context.Context, installation config.Installati
return RestoreResult{}, err
}
defer preflight.CloseArchive()
authRestoreRequired := installation.HasRuntimeAuthProjection() && manifestArchivesAuthentication(preflight.Manifest)
if authRestoreRequired {
if err := requireAuthProjectionRestorePrivilege(); err != nil {
return result, err
}
}
checkpoint, err := deps.checkpoint(ctx, transaction, installation, CreateRequest{IncludeSecrets: true, Confirm: true})
if err != nil {
@@ -150,11 +164,24 @@ func restoreWithDependencies(ctx context.Context, installation config.Installati
return result, errors.Join(err, cleanupErr)
}
var authTransaction authProjectionRestoreTransaction
if authRestoreRequired {
if deps.beginAuthProjection == nil {
cleanupErr := deps.cleanupCheckpoint(checkpoint.Path)
return result, errors.Join(errors.New("restore authentication projection dependency is unavailable"), cleanupErr)
}
authTransaction, err = deps.beginAuthProjection(ctx, installation)
if err != nil {
cleanupErr := deps.cleanupCheckpoint(checkpoint.Path)
return result, errors.Join(errors.New("restore authentication projection could not be blocked"), cleanupErr)
}
}
state := restoreTransactionState{}
defer func() {
if state.recoveryRequired(resultErr) {
recoveryContext, cancel := boundedCleanupContext()
recoveryErr := deps.recover(recoveryContext, installation, recovery, recoveryStage, state.wasRunning)
recoveryErr := deps.recover(recoveryContext, installation, recovery, recoveryStage, state.wasRunning, authTransaction)
cancel()
if recoveryErr != nil {
resultErr = errors.Join(resultErr, fmt.Errorf("restore recovery checkpoint: %w", recoveryErr))
@@ -174,6 +201,20 @@ func restoreWithDependencies(ctx context.Context, installation config.Installati
checkpointCleanupSucceeded = false
cleanupErr = errors.Join(cleanupErr, fmt.Errorf("destroy recovery checkpoint: %w", checkpointErr))
}
authCleanupSucceeded := true
if authTransaction != nil {
if resultErr != nil && !state.mutated {
if restoreErr := authTransaction.RestorePriorIfCanonicalUnchanged(); restoreErr != nil {
authCleanupSucceeded = false
cleanupErr = errors.Join(cleanupErr, errors.New("restore authentication projection could not restore its prior selector"))
}
}
if closeErr := authTransaction.Close(); closeErr != nil {
authCleanupSucceeded = false
cleanupErr = errors.Join(cleanupErr, errors.New("restore authentication projection transaction could not be closed"))
}
authTransaction = nil
}
if !state.maintenanceAttempted {
if cleanupErr != nil {
result = RestoreResult{}
@@ -204,7 +245,7 @@ func restoreWithDependencies(ctx context.Context, installation config.Installati
}
// A failed checkpoint recovery deliberately leaves admissions blocked. Starting or
// deactivating at that point would expose an unverified, possibly partial restore.
if state.mayDeactivateMaintenance() && restartCompleted {
if state.mayDeactivateMaintenance() && restartCompleted && authCleanupSucceeded {
deactivateErr, deactivated := retryBoundedCleanup(func(cleanupContext context.Context) error {
return maintenance(cleanupContext, installation, deps.runner, false)
})
@@ -250,6 +291,11 @@ func restoreWithDependencies(ctx context.Context, installation config.Installati
if err := deps.resetAuthenticationState(ctx, installation, deps.runner); err != nil {
return result, fmt.Errorf("reset authentication state: %w", err)
}
if authTransaction != nil {
if err := publishRestoredAuthentication(authTransaction); err != nil {
return result, err
}
}
if state.wasRunning {
if err := composeStartAndVerify(ctx, installation, deps.runner); err != nil {
return result, err
@@ -265,6 +311,26 @@ func restoreWithDependencies(ctx context.Context, installation config.Installati
return result, nil
}
func manifestArchivesAuthentication(manifest Manifest) bool {
for _, entry := range manifest.Entries {
if entry.Archived && entry.Kind == EntryExternalSecret && entry.Owner == "authentication-configuration" {
return true
}
}
return false
}
func publishRestoredAuthentication(transaction authProjectionRestoreTransaction) error {
status, err := transaction.PublishCanonical()
if err != nil || status.State != "ready" || !status.Equal {
if err != nil {
return fmt.Errorf("publish restored authentication projection: %w", err)
}
return errors.New("publish restored authentication projection")
}
return nil
}
func ensureCombinedRestoreCapacity(candidate, recovery PreflightResult) error {
if candidate.freeBytes == nil || candidate.stagingRoot == "" || candidate.stagingRoot != recovery.stagingRoot {
return errors.New("candidate and recovery archives do not share controlled restore staging")
+22 -6
View File
@@ -51,9 +51,20 @@ func productionRestoreDependencies(installation config.Installation) restoreDepe
},
cleanupCheckpoint: cleanupRecoveryCheckpoint,
acquireTransaction: lifecycle.AcquireTransaction,
runner: runner,
sleep: time.Sleep,
restoreFile: restoreFilePayload,
beginAuthProjection: func(ctx context.Context, target config.Installation) (authProjectionRestoreTransaction, error) {
projection := target.RuntimeAuthProjection()
if projection == nil {
return nil, errors.New("runtime authentication projection is unavailable")
}
return authconfig.BeginExternalProjectionTransaction(ctx, target.AuthenticationDirectory(), authconfig.ProjectionSpec{
RuntimeRoot: projection.Directory,
UID: projection.UID,
GID: projection.GID,
})
},
runner: runner,
sleep: time.Sleep,
restoreFile: restoreFilePayload,
restoreVolume: func(ctx context.Context, _ config.Installation, volume VolumeMetadata, input io.Reader) error {
result, err := runner.Stream(ctx, volumeRestoreCommand(volume.Name), input, io.Discard)
if err != nil || result.ExitCode != 0 {
@@ -75,8 +86,8 @@ func productionRestoreDependencies(installation config.Installation) restoreDepe
deps.prepareRecovery = func(ctx context.Context, target config.Installation, path string) (PreflightResult, error) {
return deps.preflight(ctx, target, PreflightRequest{Archive: path, Confirm: true, AllowExternalSecrets: true})
}
deps.recover = func(ctx context.Context, target config.Installation, recovery PreflightResult, staged *stagedArchive, wasRunning bool) error {
return recoverRestoreTransaction(ctx, target, recovery, staged, wasRunning, deps)
deps.recover = func(ctx context.Context, target config.Installation, recovery PreflightResult, staged *stagedArchive, wasRunning bool, transaction authProjectionRestoreTransaction) error {
return recoverRestoreTransaction(ctx, target, recovery, staged, wasRunning, deps, transaction)
}
return deps
}
@@ -88,7 +99,7 @@ func cleanupRecoveryCheckpoint(path string) error {
return nil
}
func recoverRestoreTransaction(ctx context.Context, installation config.Installation, recovery PreflightResult, staged *stagedArchive, wasRunning bool, deps restoreDependencies) (resultErr error) {
func recoverRestoreTransaction(ctx context.Context, installation config.Installation, recovery PreflightResult, staged *stagedArchive, wasRunning bool, deps restoreDependencies, transaction authProjectionRestoreTransaction) (resultErr error) {
if staged == nil || staged.file == nil {
return errors.New("recovery checkpoint was not staged before restore mutation")
}
@@ -109,6 +120,11 @@ func recoverRestoreTransaction(ctx context.Context, installation config.Installa
if err := deps.resetAuthenticationState(ctx, installation, deps.runner); err != nil {
return errors.Join(resultErr, err)
}
if transaction != nil {
if err := publishRestoredAuthentication(transaction); err != nil {
return errors.Join(resultErr, err)
}
}
if wasRunning {
if err := composeStartAndVerify(ctx, installation, deps.runner); err != nil {
resultErr = errors.Join(resultErr, err)
@@ -0,0 +1,17 @@
//go:build linux
package backup
import (
"errors"
"os"
)
var restoreProjectionEffectiveUID = os.Geteuid
func requireAuthProjectionRestorePrivilege() error {
if restoreProjectionEffectiveUID() != 0 {
return errors.New("projected authentication restore requires root")
}
return nil
}
@@ -0,0 +1,42 @@
//go:build linux
package backup
import (
"context"
"io"
"testing"
"github.com/aritmolab/thothii/tools/tht/internal/config"
"github.com/aritmolab/thothii/tools/tht/internal/lifecycle"
)
func TestRestoreAuthBearingArchiveRefusesNonRootBeforeTransactionOrWrite(t *testing.T) {
installation, archive := projectedRestoreFixture(t)
deps := restoreTestDependencies(t, newBackupRunner(installation, false))
checkpointCalled := false
beginCalled := false
writeCalled := false
deps.checkpoint = func(context.Context, *lifecycle.Transaction, config.Installation, CreateRequest) (Result, error) {
checkpointCalled = true
return Result{}, nil
}
deps.beginAuthProjection = func(context.Context, config.Installation) (authProjectionRestoreTransaction, error) {
beginCalled = true
return nil, nil
}
deps.restoreFile = func(context.Context, config.Installation, ArchiveEntryMetadata, io.Reader) error {
writeCalled = true
return nil
}
previous := restoreProjectionEffectiveUID
restoreProjectionEffectiveUID = func() int { return 1000 }
t.Cleanup(func() { restoreProjectionEffectiveUID = previous })
if _, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: archive, Confirm: true}, deps); err == nil {
t.Fatal("projected authentication restore unexpectedly accepted non-root execution")
}
if checkpointCalled || beginCalled || writeCalled {
t.Fatalf("non-root restore crossed mutation boundary: checkpoint=%t begin=%t write=%t", checkpointCalled, beginCalled, writeCalled)
}
}
@@ -0,0 +1,9 @@
//go:build !linux
package backup
import "errors"
func requireAuthProjectionRestorePrivilege() error {
return errors.New("projected authentication restore is unsupported")
}
+253 -18
View File
@@ -13,6 +13,7 @@ import (
"testing"
"time"
"github.com/aritmolab/thothii/tools/tht/internal/authconfig"
"github.com/aritmolab/thothii/tools/tht/internal/compose"
"github.com/aritmolab/thothii/tools/tht/internal/config"
"github.com/aritmolab/thothii/tools/tht/internal/lifecycle"
@@ -624,7 +625,7 @@ func TestRestoreLifecycleLockExcludesCompetingTransactionsUntilTerminalCleanup(t
}
return targetFailure
}
deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool) error {
deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool, authProjectionRestoreTransaction) error {
if err := gate("recovery"); err != nil {
return err
}
@@ -645,7 +646,7 @@ func TestRestoreLifecycleLockExcludesCompetingTransactionsUntilTerminalCleanup(t
}
return targetFailure
}
deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool) error {
deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool, authProjectionRestoreTransaction) error {
if err := gate("recovery-failure"); err != nil {
return err
}
@@ -666,7 +667,7 @@ func TestRestoreLifecycleLockExcludesCompetingTransactionsUntilTerminalCleanup(t
cancel()
return context.Canceled
}
deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool) error {
deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool, authProjectionRestoreTransaction) error {
if err := gate("recovery"); err != nil {
return err
}
@@ -892,7 +893,7 @@ func TestRestoreCannotApplyAStaleCheckpointOverAnInterleavedRestore(t *testing.T
firstDeps.restoreFile = func(context.Context, config.Installation, ArchiveEntryMetadata, io.Reader) error {
return errors.New("first target mutation failed before changing state")
}
firstDeps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool) error {
firstDeps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool, authProjectionRestoreTransaction) error {
recoveryObserved = targetState
targetState = checkpointState
firstRunner.running, firstRunner.coreRunning = true, true
@@ -1095,7 +1096,7 @@ func TestRestoreFileFailureRollsBackSecretAwareCheckpointBeforeCleanup(t *testin
return Result{Path: "/tmp/recovery.zip"}, nil
}
var events []string
deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool) error {
deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool, authProjectionRestoreTransaction) error {
events = append(events, "recover")
return nil
}
@@ -1135,7 +1136,7 @@ func TestRestoreFailureAfterAuthenticationMutationRollsBackAndClearsRuntimeState
events = append(events, "auth-runtime-reset-failed")
return resetErr
}
deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool) error {
deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool, authProjectionRestoreTransaction) error {
events = append(events, "secret-aware-recovery-and-reauth-reset")
return nil
}
@@ -1161,7 +1162,7 @@ func TestRestoreCleanupFailureDoesNotSuppressRollback(t *testing.T) {
cleanupErr := errors.New("checkpoint cleanup failure")
recovered := false
deps.restoreFile = func(context.Context, config.Installation, ArchiveEntryMetadata, io.Reader) error { return mutationErr }
deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool) error {
deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool, authProjectionRestoreTransaction) error {
recovered = true
return nil
}
@@ -1223,7 +1224,7 @@ func TestRestoreStartFailureRecoversPreviouslyRunningTarget(t *testing.T) {
backingRunner := newBackupRunner(installation, true)
deps := restoreTestDependencies(t, failStartRestoreRunner{fakeBackupRunner: backingRunner})
recovered := false
deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool) error {
deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool, authProjectionRestoreTransaction) error {
recovered = true
backingRunner.running = true
return nil
@@ -1246,7 +1247,7 @@ func TestRestoreVerificationFailureRecoversPreviouslyRunningTarget(t *testing.T)
verificationErr := errors.New("Pi is unavailable")
deps.verify["pi"] = func(context.Context, config.Installation, archiveRunner) error { return verificationErr }
recovered := false
deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool) error {
deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool, authProjectionRestoreTransaction) error {
recovered = true
return nil
}
@@ -1352,7 +1353,7 @@ func TestRestoreRecoversBehindBarrierForEveryVerificationFailure(t *testing.T) {
}
var recoveryBarrierActive bool
var recoveryContext cleanupContextObservation
deps.recover = func(ctx context.Context, _ config.Installation, _ PreflightResult, _ *stagedArchive, _ bool) error {
deps.recover = func(ctx context.Context, _ config.Installation, _ PreflightResult, _ *stagedArchive, _ bool, _ authProjectionRestoreTransaction) error {
recoveryContext = observeCleanupContext(ctx)
recoveryBarrierActive = runner.maintenance
runner.running, runner.coreRunning = true, true
@@ -1392,7 +1393,7 @@ func TestRestoreDoesNotRollbackAfterFinalDeactivationResponseLoss(t *testing.T)
}
deps := restoreTestDependencies(t, runner)
recoveryCalls := 0
deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool) error {
deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool, authProjectionRestoreTransaction) error {
recoveryCalls++
return nil
}
@@ -1450,7 +1451,7 @@ func TestRestoreUsesBoundedRecoveryContextAfterPostMutationCancellation(t *testi
}
var recoveryContext cleanupContextObservation
var recoveryBarrierActive bool
deps.recover = func(ctx context.Context, _ config.Installation, _ PreflightResult, _ *stagedArchive, _ bool) error {
deps.recover = func(ctx context.Context, _ config.Installation, _ PreflightResult, _ *stagedArchive, _ bool, _ authProjectionRestoreTransaction) error {
recoveryContext = observeCleanupContext(ctx)
recoveryBarrierActive = runner.maintenance
runner.running, runner.coreRunning = true, true
@@ -1536,7 +1537,7 @@ func TestRecoverRestoreTransactionVerifiesRecoveredStateBeforeReturning(t *testi
}
staged := stageRecoveryForTest(t, installation, recovery)
if err := recoverRestoreTransaction(context.Background(), installation, recovery, staged, true, deps); err != nil {
if err := recoverRestoreTransaction(context.Background(), installation, recovery, staged, true, deps, nil); err != nil {
t.Fatal(err)
}
if got, want := checks, []string{"health", "doctor", "pi", "workspace"}; !equalStrings(got, want) {
@@ -1566,7 +1567,7 @@ func TestRecoverRestoreTransactionFailsClosedForEveryVerification(t *testing.T)
}
staged := stageRecoveryForTest(t, installation, recovery)
err = recoverRestoreTransaction(context.Background(), installation, recovery, staged, true, deps)
err = recoverRestoreTransaction(context.Background(), installation, recovery, staged, true, deps, nil)
if !errors.Is(err, verificationErr) {
t.Fatalf("recoverRestoreTransaction() error = %v, want %v", err, verificationErr)
}
@@ -1646,8 +1647,8 @@ func TestRestoreReleasesBarrierOnlyAfterVerifiedRecoveryFromLostResponse(t *test
}
return nil
}
deps.recover = func(ctx context.Context, target config.Installation, checkpoint PreflightResult, staged *stagedArchive, wasRunning bool) error {
return recoverRestoreTransaction(ctx, target, checkpoint, staged, wasRunning, deps)
deps.recover = func(ctx context.Context, target config.Installation, checkpoint PreflightResult, staged *stagedArchive, wasRunning bool, transaction authProjectionRestoreTransaction) error {
return recoverRestoreTransaction(ctx, target, checkpoint, staged, wasRunning, deps, transaction)
}
_, err = restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: archive, Confirm: true}, deps)
@@ -1744,7 +1745,7 @@ func TestRestoreCleansMaintenanceAfterMutationAndRollbackFailures(t *testing.T)
deps.restoreFile = func(context.Context, config.Installation, ArchiveEntryMetadata, io.Reader) error {
return mutationErr
}
deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool) error {
deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool, authProjectionRestoreTransaction) error {
if test.recoveryErr == nil {
backing.running, backing.coreRunning = true, true
}
@@ -2005,7 +2006,9 @@ func restoreTestDependencies(t *testing.T, runner archiveRunner) restoreDependen
prepareRecovery: func(ctx context.Context, installation config.Installation, _ string) (PreflightResult, error) {
return Preflight(ctx, installation, PreflightRequest{Archive: recoveryArchive, Confirm: true, AllowExternalSecrets: true}, permissivePreflightDependencies())
},
recover: func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool) error { return nil },
recover: func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool, authProjectionRestoreTransaction) error {
return nil
},
cleanupCheckpoint: func(string) error { return nil },
acquireTransaction: lifecycle.AcquireTransaction,
runner: runner,
@@ -2025,3 +2028,235 @@ func restoreTestDependencies(t *testing.T, runner archiveRunner) restoreDependen
},
}
}
type projectionRestoreStub struct {
events *[]string
blocked bool
publishErr error
restoreErr error
closeErr error
}
func (stub *projectionRestoreStub) PublishCanonical() (authconfig.ProjectionStatus, error) {
*stub.events = append(*stub.events, "publish")
if stub.publishErr != nil {
return authconfig.ProjectionStatus{}, stub.publishErr
}
stub.blocked = false
return authconfig.ProjectionStatus{State: "ready", Generation: "g", CanonicalRevision: "sha256:g", Equal: true}, nil
}
func (stub *projectionRestoreStub) RestorePriorIfCanonicalUnchanged() error {
*stub.events = append(*stub.events, "restore-prior")
if stub.restoreErr != nil {
return stub.restoreErr
}
stub.blocked = false
return nil
}
func (stub *projectionRestoreStub) Close() error {
*stub.events = append(*stub.events, "close")
return stub.closeErr
}
type restartEventRunner struct {
archiveRunner
events *[]string
}
func (runner restartEventRunner) Run(ctx context.Context, args []string, input io.Reader) (compose.Result, error) {
if strings.HasSuffix(strings.Join(args, " "), " start") {
*runner.events = append(*runner.events, "restart")
}
return runner.archiveRunner.Run(ctx, args, input)
}
func (runner restartEventRunner) Stream(ctx context.Context, args []string, input io.Reader, output io.Writer) (compose.Result, error) {
return runner.archiveRunner.Stream(ctx, args, input, output)
}
func (runner restartEventRunner) SessionInventoryScope() string {
return runner.archiveRunner.SessionInventoryScope()
}
func projectedRestoreFixture(t *testing.T) (config.Installation, string) {
t.Helper()
installation := preflightTestInstallation(t)
authRoot := filepath.Join(t.TempDir(), "canonical-auth")
if err := os.Mkdir(authRoot, 0o700); err != nil {
t.Fatal(err)
}
installation.Authentication.ConfigDirectory = authRoot
installation.Authentication.RuntimeProjection = &config.RuntimeProjection{Directory: filepath.Join(t.TempDir(), "runtime-auth"), UID: 10001, GID: 10001}
archive := filepath.Join(t.TempDir(), "auth-restore.zip")
writePreflightArchive(t, archive, preflightArchiveSpec{includeSecrets: true, entries: []preflightArchiveEntry{{
path: "authentication-secrets/000-auth.yaml", body: []byte("candidate auth\n"), kind: EntryExternalSecret,
sensitive: true, owner: "authentication-configuration", sourcePath: filepath.Join(authRoot, "auth.yaml"),
}}})
return installation, archive
}
func assertOrderedEvents(t *testing.T, events []string, wants ...string) {
t.Helper()
at := 0
for _, want := range wants {
for at < len(events) && events[at] != want {
at++
}
if at == len(events) {
t.Fatalf("events = %v, want ordered subsequence %v", events, wants)
}
at++
}
}
func TestRestoreAuthBearingArchiveBlocksBeforeWritePublishesBeforeRestart(t *testing.T) {
installation, archive := projectedRestoreFixture(t)
var events []string
backing := newBackupRunner(installation, true)
runner := restartEventRunner{archiveRunner: backing, events: &events}
deps := restoreTestDependencies(t, runner)
transaction := &projectionRestoreStub{events: &events}
deps.beginAuthProjection = func(context.Context, config.Installation) (authProjectionRestoreTransaction, error) {
transaction.blocked = true
events = append(events, "begin")
return transaction, nil
}
deps.restoreFile = func(_ context.Context, _ config.Installation, entry ArchiveEntryMetadata, _ io.Reader) error {
if entry.Owner == "authentication-configuration" {
if !transaction.blocked {
t.Fatal("auth destination write began without blocked projection")
}
events = append(events, "restore-auth")
}
return nil
}
if _, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: archive, Confirm: true}, deps); err != nil {
t.Fatal(err)
}
assertOrderedEvents(t, events, "begin", "restore-auth", "publish", "restart", "close")
if transaction.blocked {
t.Fatalf("successful restore closed while projection remained blocked: %v", events)
}
}
func TestRestoreAuthCandidatePublicationFailureRecoversThenStaysBlockedWithoutRestart(t *testing.T) {
installation, archive := projectedRestoreFixture(t)
var events []string
backing := newBackupRunner(installation, true)
runner := restartEventRunner{archiveRunner: backing, events: &events}
deps := restoreTestDependencies(t, runner)
publicationErr := errors.New("synthetic publication failure")
transaction := &projectionRestoreStub{events: &events, publishErr: publicationErr}
deps.beginAuthProjection = func(context.Context, config.Installation) (authProjectionRestoreTransaction, error) {
transaction.blocked = true
return transaction, nil
}
deps.recover = func(_ context.Context, _ config.Installation, _ PreflightResult, _ *stagedArchive, _ bool, transaction authProjectionRestoreTransaction) error {
events = append(events, "restore-checkpoint")
_, err := transaction.PublishCanonical()
return err
}
if _, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: archive, Confirm: true}, deps); !errors.Is(err, publicationErr) {
t.Fatalf("restore error = %v, want publication failure", err)
}
assertOrderedEvents(t, events, "publish", "restore-checkpoint", "close")
if !transaction.blocked {
t.Fatal("publication failure reopened projection")
}
if backing.startCount != 0 {
t.Fatalf("restart after failed candidate/recovery publication = %d", backing.startCount)
}
if !backing.maintenance {
t.Fatal("admissions reopened after failed recovery publication")
}
}
func TestRestoreAuthVerificationFailuresRepublishCheckpointBeforeRecoveryRestart(t *testing.T) {
for _, failed := range []string{"health", "doctor", "pi", "workspace"} {
t.Run(failed, func(t *testing.T) {
installation, archive := projectedRestoreFixture(t)
var events []string
backing := newBackupRunner(installation, true)
runner := restartEventRunner{archiveRunner: backing, events: &events}
deps := restoreTestDependencies(t, runner)
transaction := &projectionRestoreStub{events: &events}
deps.beginAuthProjection = func(context.Context, config.Installation) (authProjectionRestoreTransaction, error) {
transaction.blocked = true
return transaction, nil
}
for _, name := range []string{"health", "doctor", "pi", "workspace"} {
name := name
deps.verify[name] = func(context.Context, config.Installation, archiveRunner) error {
events = append(events, "candidate-"+name)
if name == failed {
return errors.New("synthetic " + name + " failure")
}
return nil
}
}
deps.recover = func(ctx context.Context, target config.Installation, _ PreflightResult, _ *stagedArchive, wasRunning bool, transaction authProjectionRestoreTransaction) error {
events = append(events, "restore-checkpoint")
if _, err := transaction.PublishCanonical(); err != nil {
return err
}
events = append(events, "verify-checkpoint")
if wasRunning {
return composeStartAndVerify(ctx, target, runner)
}
return nil
}
if _, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: archive, Confirm: true}, deps); err == nil {
t.Fatalf("%s failure accepted", failed)
}
assertOrderedEvents(t, events, "publish", "candidate-"+failed, "restore-checkpoint", "publish", "verify-checkpoint", "restart", "close")
if transaction.blocked {
t.Fatalf("verified checkpoint recovery remained blocked: %v", events)
}
})
}
}
func TestRestoreAuthPreMutationFailureRestoresPriorReadySelector(t *testing.T) {
installation, archive := projectedRestoreFixture(t)
var events []string
backing := newBackupRunner(installation, false)
runner := &commandFailureRunner{fakeBackupRunner: backing, failures: []*commandFailure{{
match: func(command string) bool { return strings.Contains(command, " ps --all --format json") },
err: errors.New("synthetic pre-mutation failure"), remaining: 1,
}}}
deps := restoreTestDependencies(t, runner)
transaction := &projectionRestoreStub{events: &events}
deps.beginAuthProjection = func(context.Context, config.Installation) (authProjectionRestoreTransaction, error) {
transaction.blocked = true
return transaction, nil
}
deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool, authProjectionRestoreTransaction) error {
t.Fatal("recovery ran before any destination mutation")
return nil
}
if _, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: archive, Confirm: true}, deps); err == nil {
t.Fatal("pre-mutation failure accepted")
}
assertOrderedEvents(t, events, "restore-prior", "close")
if transaction.blocked {
t.Fatal("unchanged canonical pre-write failure did not restore ready selector")
}
}
func TestRestoreNonAuthArchiveNeverBeginsProjection(t *testing.T) {
installation := preflightTestInstallation(t)
deps := restoreTestDependencies(t, newBackupRunner(installation, false))
called := false
deps.beginAuthProjection = func(context.Context, config.Installation) (authProjectionRestoreTransaction, error) {
called = true
return nil, errors.New("must not begin")
}
if _, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: restoreArchive(t), Confirm: true}, deps); err != nil {
t.Fatal(err)
}
if called {
t.Fatal("non-auth archive began projection transaction")
}
}