mptyl/ThothII
1
0
Fork 0
Code Issues 25 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

fix(auth): close Task 15 review round two

Browse Source
This commit is contained in:
marcopan
2026-08-18 07:21:24 +02:00
parent 225ffc8e20
commit fe190e7046
10 changed files with 305 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/tht/internal/backup/restore_test.go
+105 -22
View File
@@ -120,14 +120,15 @@ func TestRestoreStoppedInstallationRunsCheckpointRestoreAndVerification(t *testi
var events []string
var checkpointRequest CreateRequest
deps := restoreTestDependencies(t, runner)
prepareRecovery := deps.prepareRecovery
deps.checkpoint = func(_ context.Context, _ *lifecycle.Transaction, _ config.Installation, request CreateRequest) (Result, error) {
events = append(events, "checkpoint")
checkpointRequest = request
return Result{Path: "/tmp/checkpoint.zip"}, nil
}
deps.prepareRecovery = func(context.Context, config.Installation, string) (PreflightResult, error) {
deps.prepareRecovery = func(ctx context.Context, target config.Installation, path string) (PreflightResult, error) {
events = append(events, "prepare-recovery")
return PreflightResult{}, nil
return prepareRecovery(ctx, target, path)
}
deps.cleanupCheckpoint = func(string) error {
events = append(events, "cleanup-checkpoint")
@@ -167,6 +168,49 @@ func TestRestoreStoppedInstallationRunsCheckpointRestoreAndVerification(t *testi
}
}
func TestRestoreRejectsCombinedCandidateAndRecoveryStagingCapacityBeforeMutation(t *testing.T) {
installation := preflightTestInstallation(t)
candidateArchive := restoreArchive(t)
recoveryArchive := restoreArchive(t)
runner := newBackupRunner(installation, false)
dependencies := restoreTestDependencies(t, runner)
capacityChecks := 0
preflightDependencies := permissivePreflightDependencies()
preflightDependencies.FreeBytes = func(string) (uint64, error) {
capacityChecks++
if capacityChecks == 5 {
return 0, nil
}
return 1 << 30, nil
}
dependencies.preflight = func(ctx context.Context, target config.Installation, request PreflightRequest) (PreflightResult, error) {
return Preflight(ctx, target, request, preflightDependencies)
}
dependencies.checkpoint = func(context.Context, *lifecycle.Transaction, config.Installation, CreateRequest) (Result, error) {
return Result{Path: recoveryArchive}, nil
}
dependencies.prepareRecovery = func(ctx context.Context, target config.Installation, path string) (PreflightResult, error) {
return Preflight(ctx, target, PreflightRequest{Archive: path, Confirm: true, AllowExternalSecrets: true}, preflightDependencies)
}
mutated := false
dependencies.restoreFile = func(context.Context, config.Installation, ArchiveEntryMetadata, io.Reader) error {
mutated = true
return nil
}
_, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: candidateArchive, Confirm: true}, dependencies)
if err == nil || !strings.Contains(err.Error(), "staging") {
t.Fatalf("Restore() error = %v, want recovery staging capacity rejection", err)
}
if mutated {
t.Fatal("restore mutated the installation before reserving candidate and recovery staging capacity")
}
if capacityChecks != 5 {
t.Fatalf("free-space checks = %d, want candidate/recovery preflight and staging checks", capacityChecks)
}
}
func TestRestoreClosesTargetArchiveBeforeReleasingLifecycleLock(t *testing.T) {
installation := preflightTestInstallation(t)
archive := filepath.Join(t.TempDir(), "restore.zip")
@@ -301,7 +345,7 @@ func TestRestoreLifecycleLockExcludesCompetingTransactionsUntilTerminalCleanup(t
gate("target-failure")
return targetFailure
}
deps.recover = func(context.Context, config.Installation, PreflightResult, bool) error {
deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool) error {
gate("recovery")
runner.running, runner.coreRunning = true, true
return nil
@@ -318,7 +362,7 @@ func TestRestoreLifecycleLockExcludesCompetingTransactionsUntilTerminalCleanup(t
gate("target-failure")
return targetFailure
}
deps.recover = func(context.Context, config.Installation, PreflightResult, bool) error {
deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool) error {
gate("recovery-failure")
return recoveryFailure
}
@@ -335,7 +379,7 @@ func TestRestoreLifecycleLockExcludesCompetingTransactionsUntilTerminalCleanup(t
cancel()
return context.Canceled
}
deps.recover = func(context.Context, config.Installation, PreflightResult, bool) error {
deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool) error {
gate("recovery")
runner.running, runner.coreRunning = true, true
return nil
@@ -353,6 +397,11 @@ func TestRestoreLifecycleLockExcludesCompetingTransactionsUntilTerminalCleanup(t
entries: []preflightArchiveEntry{{path: "configuration/operator.env", body: []byte("safe")}},
})
runner := &lifecycleGateRunner{fakeBackupRunner: newBackupRunner(installation, true)}
recoveryArchive := filepath.Join(t.TempDir(), "recovery.zip")
writePreflightArchive(t, recoveryArchive, preflightArchiveSpec{
installationID: fixture.installationID,
entries: []preflightArchiveEntry{{path: "configuration/operator.env", body: []byte("checkpoint")}},
})
stages := make(chan string)
continueStage := make(chan struct{})
gate := func(stage string) {
@@ -363,6 +412,9 @@ func TestRestoreLifecycleLockExcludesCompetingTransactionsUntilTerminalCleanup(t
caller, cancel := context.WithCancel(context.Background())
defer cancel()
deps := restoreTestDependencies(t, runner)
deps.prepareRecovery = func(ctx context.Context, target config.Installation, _ string) (PreflightResult, error) {
return Preflight(ctx, target, PreflightRequest{Archive: recoveryArchive, Confirm: true, AllowExternalSecrets: true}, permissivePreflightDependencies())
}
deps.acquireTransaction = lifecycle.AcquireTransaction
deps.checkpoint = func(context.Context, *lifecycle.Transaction, config.Installation, CreateRequest) (Result, error) {
gate("checkpoint")
@@ -474,6 +526,14 @@ func TestRestoreCannotApplyAStaleCheckpointOverAnInterleavedRestore(t *testing.T
continueCheckpoint := make(chan struct{})
firstRunner := newBackupRunner(installation, true)
firstDeps := restoreTestDependencies(t, firstRunner)
recoveryArchive := filepath.Join(t.TempDir(), "first-recovery.zip")
writePreflightArchive(t, recoveryArchive, preflightArchiveSpec{
installationID: fixture.installationID,
entries: []preflightArchiveEntry{{path: "configuration/operator.env", body: []byte("checkpoint")}},
})
firstDeps.prepareRecovery = func(ctx context.Context, target config.Installation, _ string) (PreflightResult, error) {
return Preflight(ctx, target, PreflightRequest{Archive: recoveryArchive, Confirm: true, AllowExternalSecrets: true}, permissivePreflightDependencies())
}
firstDeps.acquireTransaction = lifecycle.AcquireTransaction
firstDeps.checkpoint = func(context.Context, *lifecycle.Transaction, config.Installation, CreateRequest) (Result, error) {
checkpointState = targetState
@@ -484,7 +544,7 @@ func TestRestoreCannotApplyAStaleCheckpointOverAnInterleavedRestore(t *testing.T
firstDeps.restoreFile = func(context.Context, config.Installation, ArchiveEntryMetadata, io.Reader) error {
return errors.New("first target mutation failed before changing state")
}
firstDeps.recover = func(context.Context, config.Installation, PreflightResult, bool) error {
firstDeps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool) error {
recoveryObserved = targetState
targetState = checkpointState
firstRunner.running, firstRunner.coreRunning = true, true
@@ -661,7 +721,7 @@ func TestRestoreFileFailureRollsBackSecretAwareCheckpointBeforeCleanup(t *testin
return Result{Path: "/tmp/recovery.zip"}, nil
}
var events []string
deps.recover = func(context.Context, config.Installation, PreflightResult, bool) error {
deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool) error {
events = append(events, "recover")
return nil
}
@@ -701,7 +761,7 @@ func TestRestoreFailureAfterAuthenticationMutationRollsBackAndClearsRuntimeState
events = append(events, "auth-runtime-reset-failed")
return resetErr
}
deps.recover = func(context.Context, config.Installation, PreflightResult, bool) error {
deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool) error {
events = append(events, "secret-aware-recovery-and-reauth-reset")
return nil
}
@@ -727,7 +787,10 @@ func TestRestoreCleanupFailureDoesNotSuppressRollback(t *testing.T) {
cleanupErr := errors.New("checkpoint cleanup failure")
recovered := false
deps.restoreFile = func(context.Context, config.Installation, ArchiveEntryMetadata, io.Reader) error { return mutationErr }
deps.recover = func(context.Context, config.Installation, PreflightResult, bool) error { recovered = true; return nil }
deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool) error {
recovered = true
return nil
}
deps.cleanupCheckpoint = func(string) error { return cleanupErr }
_, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: archive, Confirm: true}, deps)
@@ -783,7 +846,7 @@ func TestRestoreStartFailureRecoversPreviouslyRunningTarget(t *testing.T) {
backingRunner := newBackupRunner(installation, true)
deps := restoreTestDependencies(t, failStartRestoreRunner{fakeBackupRunner: backingRunner})
recovered := false
deps.recover = func(context.Context, config.Installation, PreflightResult, bool) error {
deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool) error {
recovered = true
backingRunner.running = true
return nil
@@ -806,7 +869,7 @@ func TestRestoreVerificationFailureRecoversPreviouslyRunningTarget(t *testing.T)
verificationErr := errors.New("Pi is unavailable")
deps.verify["pi"] = func(context.Context, config.Installation, archiveRunner) error { return verificationErr }
recovered := false
deps.recover = func(context.Context, config.Installation, PreflightResult, bool) error {
deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool) error {
recovered = true
return nil
}
@@ -884,7 +947,7 @@ func TestRestoreRecoversBehindBarrierForEveryVerificationFailure(t *testing.T) {
}
var recoveryBarrierActive bool
var recoveryContext cleanupContextObservation
deps.recover = func(ctx context.Context, _ config.Installation, _ PreflightResult, _ bool) error {
deps.recover = func(ctx context.Context, _ config.Installation, _ PreflightResult, _ *stagedArchive, _ bool) error {
recoveryContext = observeCleanupContext(ctx)
recoveryBarrierActive = runner.maintenance
runner.running, runner.coreRunning = true, true
@@ -924,7 +987,7 @@ func TestRestoreDoesNotRollbackAfterFinalDeactivationResponseLoss(t *testing.T)
}
deps := restoreTestDependencies(t, runner)
recoveryCalls := 0
deps.recover = func(context.Context, config.Installation, PreflightResult, bool) error {
deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool) error {
recoveryCalls++
return nil
}
@@ -982,7 +1045,7 @@ func TestRestoreUsesBoundedRecoveryContextAfterPostMutationCancellation(t *testi
}
var recoveryContext cleanupContextObservation
var recoveryBarrierActive bool
deps.recover = func(ctx context.Context, _ config.Installation, _ PreflightResult, _ bool) error {
deps.recover = func(ctx context.Context, _ config.Installation, _ PreflightResult, _ *stagedArchive, _ bool) error {
recoveryContext = observeCleanupContext(ctx)
recoveryBarrierActive = runner.maintenance
runner.running, runner.coreRunning = true, true
@@ -1067,7 +1130,8 @@ func TestRecoverRestoreTransactionVerifiesRecoveredStateBeforeReturning(t *testi
}
}
if err := recoverRestoreTransaction(context.Background(), installation, recovery, true, deps); err != nil {
staged := stageRecoveryForTest(t, installation, recovery)
if err := recoverRestoreTransaction(context.Background(), installation, recovery, staged, true, deps); err != nil {
t.Fatal(err)
}
if got, want := checks, []string{"health", "doctor", "pi", "workspace"}; !equalStrings(got, want) {
@@ -1096,7 +1160,8 @@ func TestRecoverRestoreTransactionFailsClosedForEveryVerification(t *testing.T)
return verificationErr
}
err = recoverRestoreTransaction(context.Background(), installation, recovery, true, deps)
staged := stageRecoveryForTest(t, installation, recovery)
err = recoverRestoreTransaction(context.Background(), installation, recovery, staged, true, deps)
if !errors.Is(err, verificationErr) {
t.Fatalf("recoverRestoreTransaction() error = %v, want %v", err, verificationErr)
}
@@ -1176,8 +1241,8 @@ func TestRestoreReleasesBarrierOnlyAfterVerifiedRecoveryFromLostResponse(t *test
}
return nil
}
deps.recover = func(ctx context.Context, target config.Installation, checkpoint PreflightResult, wasRunning bool) error {
return recoverRestoreTransaction(ctx, target, checkpoint, wasRunning, deps)
deps.recover = func(ctx context.Context, target config.Installation, checkpoint PreflightResult, staged *stagedArchive, wasRunning bool) error {
return recoverRestoreTransaction(ctx, target, checkpoint, staged, wasRunning, deps)
}
_, err = restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: archive, Confirm: true}, deps)
@@ -1274,7 +1339,7 @@ func TestRestoreCleansMaintenanceAfterMutationAndRollbackFailures(t *testing.T)
deps.restoreFile = func(context.Context, config.Installation, ArchiveEntryMetadata, io.Reader) error {
return mutationErr
}
deps.recover = func(context.Context, config.Installation, PreflightResult, bool) error {
deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool) error {
if test.recoveryErr == nil {
backing.running, backing.coreRunning = true, true
}
@@ -1505,8 +1570,26 @@ func equalStrings(got, want []string) bool {
return true
}
func stageRecoveryForTest(t *testing.T, installation config.Installation, recovery PreflightResult) *stagedArchive {
t.Helper()
if err := os.MkdirAll(installation.ControlDirectory(), 0o700); err != nil {
t.Fatal(err)
}
staged, err := recovery.StageArchive(context.Background())
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
if err := staged.Close(); err != nil {
t.Error(err)
}
})
return staged
}
func restoreTestDependencies(t *testing.T, runner archiveRunner) restoreDependencies {
t.Helper()
recoveryArchive := restoreArchive(t)
return restoreDependencies{
preflight: func(ctx context.Context, installation config.Installation, request PreflightRequest) (PreflightResult, error) {
return Preflight(ctx, installation, request, permissivePreflightDependencies())
@@ -1514,10 +1597,10 @@ func restoreTestDependencies(t *testing.T, runner archiveRunner) restoreDependen
checkpoint: func(context.Context, *lifecycle.Transaction, config.Installation, CreateRequest) (Result, error) {
return Result{Path: "/tmp/default-checkpoint.zip"}, nil
},
prepareRecovery: func(context.Context, config.Installation, string) (PreflightResult, error) {
return PreflightResult{}, nil
prepareRecovery: func(ctx context.Context, installation config.Installation, _ string) (PreflightResult, error) {
return Preflight(ctx, installation, PreflightRequest{Archive: recoveryArchive, Confirm: true, AllowExternalSecrets: true}, permissivePreflightDependencies())
},
recover: func(context.Context, config.Installation, PreflightResult, bool) error { return nil },
recover: func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool) error { return nil },
cleanupCheckpoint: func(string) error { return nil },
acquireTransaction: lifecycle.AcquireTransaction,
runner: runner,