mptyl/ThothII
1
0
Fork 0
Code Issues 25 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

fix(auth): serialize restore checkpoint lifecycle

Browse Source
This commit is contained in:
marcopan
2026-08-18 02:43:42 +02:00
parent 39a0fdbd00
commit 6ec5b76c54
4 changed files with 391 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/tht/internal/backup/restore.go
+46 -29
View File
@@ -33,8 +33,11 @@ type restoreLock interface{ Release() error }
type restoreVerify func(context.Context, config.Installation, archiveRunner) error
type restoreDependencies struct {
preflight func(context.Context, config.Installation, PreflightRequest) (PreflightResult, error)
checkpoint func(context.Context, config.Installation, CreateRequest) (Result, error)
preflight func(context.Context, config.Installation, PreflightRequest) (PreflightResult, error)
// checkpointLocked creates the secret-aware recovery archive while the caller already owns
// the installation lifecycle lock. It must not call public Create, which would re-acquire the
// non-reentrant lock and deadlock the restore transaction.
checkpointLocked func(context.Context, config.Installation, CreateRequest) (Result, error)
prepareRecovery func(context.Context, config.Installation, string) (PreflightResult, error)
recover func(context.Context, config.Installation, PreflightResult, bool) error
cleanupCheckpoint func(string) error
@@ -81,36 +84,25 @@ func restoreWithDependencies(ctx context.Context, installation config.Installati
if request.Archive == "" {
return RestoreResult{}, errors.New("restore archive is required")
}
if deps.preflight == nil || deps.checkpoint == nil || deps.prepareRecovery == nil || deps.recover == nil || deps.cleanupCheckpoint == nil || deps.acquireLock == nil || deps.runner == nil || deps.restoreFile == nil || deps.restoreVolume == nil || deps.resetAuthenticationState == nil || deps.verify == nil {
if deps.preflight == nil || deps.checkpointLocked == nil || deps.prepareRecovery == nil || deps.recover == nil || deps.cleanupCheckpoint == nil || deps.acquireLock == nil || deps.runner == nil || deps.restoreFile == nil || deps.restoreVolume == nil || deps.resetAuthenticationState == nil || deps.verify == nil {
return RestoreResult{}, errors.New("restore dependencies are incomplete")
}
preflight, err := deps.preflight(ctx, installation, PreflightRequest{Archive: request.Archive, Confirm: true, AllowExternalSecrets: true})
if err != nil {
return RestoreResult{}, err
}
defer preflight.CloseArchive()
archive, err := preflight.RevalidateArchive()
if err != nil {
_ = preflight.CloseArchive()
return RestoreResult{}, err
}
checkpoint, err := deps.checkpoint(ctx, installation, CreateRequest{IncludeSecrets: true, Confirm: true})
if err != nil {
return RestoreResult{}, fmt.Errorf("create recovery checkpoint: %w", err)
}
recovery, err := deps.prepareRecovery(ctx, installation, checkpoint.Path)
if err != nil {
cleanupErr := deps.cleanupCheckpoint(checkpoint.Path)
return RestoreResult{}, errors.Join(fmt.Errorf("validate recovery checkpoint: %w", err), cleanupErr)
}
defer func() {
if cleanupErr := deps.cleanupCheckpoint(checkpoint.Path); cleanupErr != nil {
resultErr = errors.Join(resultErr, fmt.Errorf("destroy recovery checkpoint: %w", cleanupErr))
}
}()
defer recovery.CloseArchive()
// Lock ordering is lifecycle lock -> Compose/operator maintenance barrier. The core operator
// command never acquires the host lifecycle lock, so this order cannot form a lock cycle with
// Docker Compose or the durable maintenance marker.
lock, err := deps.acquireLock(installation)
if err != nil {
_ = preflight.CloseArchive()
return result, err
}
defer func() {
@@ -119,12 +111,20 @@ func restoreWithDependencies(ctx context.Context, installation config.Installati
resultErr = errors.Join(resultErr, fmt.Errorf("release restore lifecycle lock: %w", releaseErr))
}
}()
defer preflight.CloseArchive()
wasRunning, err := installationRunning(ctx, installation, deps.runner)
checkpoint, err := deps.checkpointLocked(ctx, installation, CreateRequest{IncludeSecrets: true, Confirm: true})
if err != nil {
return result, err
return result, fmt.Errorf("create recovery checkpoint: %w", err)
}
state := restoreTransactionState{wasRunning: wasRunning}
recovery, err := deps.prepareRecovery(ctx, installation, checkpoint.Path)
if err != nil {
cleanupErr := deps.cleanupCheckpoint(checkpoint.Path)
return result, errors.Join(fmt.Errorf("validate recovery checkpoint: %w", err), cleanupErr)
}
defer recovery.CloseArchive()
state := restoreTransactionState{}
defer func() {
if state.recoveryRequired(resultErr) {
recoveryContext, cancel := boundedCleanupContext()
@@ -141,10 +141,27 @@ func restoreWithDependencies(ctx context.Context, installation config.Installati
state.stopAttempted = false
}
}
checkpointCleanupSucceeded := true
var cleanupErr error
if checkpointErr := deps.cleanupCheckpoint(checkpoint.Path); checkpointErr != nil {
checkpointCleanupSucceeded = false
cleanupErr = errors.Join(cleanupErr, fmt.Errorf("destroy recovery checkpoint: %w", checkpointErr))
}
if !state.maintenanceAttempted {
if cleanupErr != nil {
result = RestoreResult{}
resultErr = errors.Join(resultErr, cleanupErr)
}
return
}
// A recovery checkpoint remains secret-bearing transaction state. Do not reopen
// admissions until it has been safely deleted, even if the restored target verified.
if !checkpointCleanupSucceeded {
result = RestoreResult{}
resultErr = errors.Join(resultErr, cleanupErr)
return
}
var cleanupErr error
restartCompleted := true
if state.wasRunning && state.stopAttempted && state.mayDeactivateMaintenance() {
startErr, started := retryBoundedCleanup(func(cleanupContext context.Context) error {
@@ -177,6 +194,12 @@ func restoreWithDependencies(ctx context.Context, installation config.Installati
resultErr = errors.Join(resultErr, cleanupErr)
}
}()
wasRunning, err := installationRunning(ctx, installation, deps.runner)
if err != nil {
return result, err
}
state.wasRunning = wasRunning
if state.wasRunning {
// The activation command may take effect even when its response is lost. Track the attempt,
// not merely a successful return, so every subsequent path compensates from durable state.
@@ -213,12 +236,6 @@ func restoreWithDependencies(ctx context.Context, installation config.Installati
}
state.verified = true
result.Verified = true
if state.wasRunning {
if err := maintenance(ctx, installation, deps.runner, false); err != nil {
return result, err
}
state.maintenanceAttempted = false
}
return result, nil
}