324 lines
12 KiB
Go
324 lines
12 KiB
Go
package backup
|
|
|
|
import (
|
|
"archive/zip"
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"time"
|
|
|
|
"github.com/aritmolab/thothii/tools/tht/internal/config"
|
|
)
|
|
|
|
var ErrRestoreConfirmationRequired = errors.New("restore requires --yes")
|
|
|
|
// RestoreRequest describes the deliberately-confirmed archive restoration.
|
|
type RestoreRequest struct {
|
|
Archive string
|
|
Confirm bool
|
|
Drain bool
|
|
}
|
|
|
|
// RestoreResult records the final service state. Secret-bearing recovery checkpoints are always
|
|
// destroyed internally and are therefore never exposed to callers.
|
|
type RestoreResult struct {
|
|
// Deprecated: always empty. Recovery checkpoints are private transaction internals.
|
|
Checkpoint string
|
|
Restarted bool
|
|
Verified bool
|
|
}
|
|
|
|
type restoreLock interface{ Release() error }
|
|
type restoreVerify func(context.Context, config.Installation, archiveRunner) error
|
|
|
|
type restoreDependencies struct {
|
|
preflight func(context.Context, config.Installation, PreflightRequest) (PreflightResult, error)
|
|
checkpoint func(context.Context, config.Installation, CreateRequest) (Result, error)
|
|
prepareRecovery func(context.Context, config.Installation, string) (PreflightResult, error)
|
|
recover func(context.Context, config.Installation, PreflightResult, bool) error
|
|
cleanupCheckpoint func(string) error
|
|
acquireLock func(config.Installation) (restoreLock, error)
|
|
runner archiveRunner
|
|
sleep func(duration time.Duration)
|
|
restoreFile func(context.Context, config.Installation, ArchiveEntryMetadata, io.Reader) error
|
|
restoreVolume func(context.Context, config.Installation, VolumeMetadata, io.Reader) error
|
|
resetAuthenticationState func(context.Context, config.Installation, archiveRunner) error
|
|
verify map[string]restoreVerify
|
|
}
|
|
|
|
// restoreTransactionState is the restore admission state machine. The durable maintenance
|
|
// barrier is released only after the target archive has been verified, or after a separately
|
|
// verified checkpoint recovery. Every mutable Docker command is tracked before it is invoked.
|
|
type restoreTransactionState struct {
|
|
wasRunning bool
|
|
maintenanceAttempted bool
|
|
stopAttempted bool
|
|
mutated bool
|
|
verified bool
|
|
recovered bool
|
|
}
|
|
|
|
func (state restoreTransactionState) recoveryRequired(resultErr error) bool {
|
|
return resultErr != nil && state.mutated && !state.verified
|
|
}
|
|
|
|
func (state restoreTransactionState) mayDeactivateMaintenance() bool {
|
|
return !state.mutated || state.verified || state.recovered
|
|
}
|
|
|
|
// Restore runs the host transaction through the same concrete Docker/filesystem boundaries used
|
|
// by backup creation. The injectable core below exists only to make every failure boundary
|
|
// deterministic in tests.
|
|
func Restore(ctx context.Context, installation config.Installation, request RestoreRequest) (RestoreResult, error) {
|
|
return restoreWithDependencies(ctx, installation, request, productionRestoreDependencies(installation))
|
|
}
|
|
|
|
func restoreWithDependencies(ctx context.Context, installation config.Installation, request RestoreRequest, deps restoreDependencies) (result RestoreResult, resultErr error) {
|
|
if !request.Confirm {
|
|
return RestoreResult{}, ErrRestoreConfirmationRequired
|
|
}
|
|
if request.Archive == "" {
|
|
return RestoreResult{}, errors.New("restore archive is required")
|
|
}
|
|
if deps.preflight == nil || deps.checkpoint == nil || deps.prepareRecovery == nil || deps.recover == nil || deps.cleanupCheckpoint == nil || deps.acquireLock == nil || deps.runner == nil || deps.restoreFile == nil || deps.restoreVolume == nil || deps.resetAuthenticationState == nil || deps.verify == nil {
|
|
return RestoreResult{}, errors.New("restore dependencies are incomplete")
|
|
}
|
|
preflight, err := deps.preflight(ctx, installation, PreflightRequest{Archive: request.Archive, Confirm: true, AllowExternalSecrets: true})
|
|
if err != nil {
|
|
return RestoreResult{}, err
|
|
}
|
|
defer preflight.CloseArchive()
|
|
archive, err := preflight.RevalidateArchive()
|
|
if err != nil {
|
|
return RestoreResult{}, err
|
|
}
|
|
|
|
checkpoint, err := deps.checkpoint(ctx, installation, CreateRequest{IncludeSecrets: true, Confirm: true})
|
|
if err != nil {
|
|
return RestoreResult{}, fmt.Errorf("create recovery checkpoint: %w", err)
|
|
}
|
|
recovery, err := deps.prepareRecovery(ctx, installation, checkpoint.Path)
|
|
if err != nil {
|
|
cleanupErr := deps.cleanupCheckpoint(checkpoint.Path)
|
|
return RestoreResult{}, errors.Join(fmt.Errorf("validate recovery checkpoint: %w", err), cleanupErr)
|
|
}
|
|
defer func() {
|
|
if cleanupErr := deps.cleanupCheckpoint(checkpoint.Path); cleanupErr != nil {
|
|
resultErr = errors.Join(resultErr, fmt.Errorf("destroy recovery checkpoint: %w", cleanupErr))
|
|
}
|
|
}()
|
|
defer recovery.CloseArchive()
|
|
lock, err := deps.acquireLock(installation)
|
|
if err != nil {
|
|
return result, err
|
|
}
|
|
defer func() {
|
|
if releaseErr := lock.Release(); releaseErr != nil {
|
|
result = RestoreResult{}
|
|
resultErr = errors.Join(resultErr, fmt.Errorf("release restore lifecycle lock: %w", releaseErr))
|
|
}
|
|
}()
|
|
|
|
wasRunning, err := installationRunning(ctx, installation, deps.runner)
|
|
if err != nil {
|
|
return result, err
|
|
}
|
|
state := restoreTransactionState{wasRunning: wasRunning}
|
|
defer func() {
|
|
if state.recoveryRequired(resultErr) {
|
|
recoveryContext, cancel := boundedCleanupContext()
|
|
recoveryErr := deps.recover(recoveryContext, installation, recovery, state.wasRunning)
|
|
cancel()
|
|
if recoveryErr != nil {
|
|
resultErr = errors.Join(resultErr, fmt.Errorf("restore recovery checkpoint: %w", recoveryErr))
|
|
if recoveryReachedVerifiedState(recoveryErr) {
|
|
state.recovered = true
|
|
state.stopAttempted = false
|
|
}
|
|
} else {
|
|
state.recovered = true
|
|
state.stopAttempted = false
|
|
}
|
|
}
|
|
if !state.maintenanceAttempted {
|
|
return
|
|
}
|
|
var cleanupErr error
|
|
restartCompleted := true
|
|
if state.wasRunning && state.stopAttempted && state.mayDeactivateMaintenance() {
|
|
startErr, started := retryBoundedCleanup(func(cleanupContext context.Context) error {
|
|
return composeStartAndVerify(cleanupContext, installation, deps.runner)
|
|
})
|
|
if startErr != nil {
|
|
cleanupErr = errors.Join(cleanupErr, fmt.Errorf("restore maintenance cleanup restart: %w", startErr))
|
|
}
|
|
if started {
|
|
state.stopAttempted = false
|
|
} else {
|
|
restartCompleted = false
|
|
}
|
|
}
|
|
// A failed checkpoint recovery deliberately leaves admissions blocked. Starting or
|
|
// deactivating at that point would expose an unverified, possibly partial restore.
|
|
if state.mayDeactivateMaintenance() && restartCompleted {
|
|
deactivateErr, deactivated := retryBoundedCleanup(func(cleanupContext context.Context) error {
|
|
return maintenance(cleanupContext, installation, deps.runner, false)
|
|
})
|
|
if deactivateErr != nil {
|
|
cleanupErr = errors.Join(cleanupErr, fmt.Errorf("restore maintenance cleanup: %w", deactivateErr))
|
|
}
|
|
if deactivated {
|
|
state.maintenanceAttempted = false
|
|
}
|
|
}
|
|
if cleanupErr != nil {
|
|
result = RestoreResult{}
|
|
resultErr = errors.Join(resultErr, cleanupErr)
|
|
}
|
|
}()
|
|
if state.wasRunning {
|
|
// The activation command may take effect even when its response is lost. Track the attempt,
|
|
// not merely a successful return, so every subsequent path compensates from durable state.
|
|
state.maintenanceAttempted = true
|
|
if err := maintenance(ctx, installation, deps.runner, true); err != nil {
|
|
return result, err
|
|
}
|
|
if err := waitForNoActiveSessions(ctx, installation, deps.runner, request.Drain, deps.sleep); err != nil {
|
|
return result, err
|
|
}
|
|
// Compose may stop the core and then lose its response. Cleanup must therefore restart after
|
|
// any stop attempt, including a command that returns an error.
|
|
state.stopAttempted = true
|
|
if err := runCompose(ctx, installation, deps.runner, "stop"); err != nil {
|
|
return result, err
|
|
}
|
|
}
|
|
state.mutated = true
|
|
if err := restoreVerifiedEntries(ctx, installation, preflight, archive, deps.restoreFile, deps.restoreVolume); err != nil {
|
|
return result, err
|
|
}
|
|
if err := deps.resetAuthenticationState(ctx, installation, deps.runner); err != nil {
|
|
return result, fmt.Errorf("reset authentication state: %w", err)
|
|
}
|
|
if state.wasRunning {
|
|
if err := composeStartAndVerify(ctx, installation, deps.runner); err != nil {
|
|
return result, err
|
|
}
|
|
state.stopAttempted = false
|
|
result.Restarted = true
|
|
}
|
|
if err := verifyRestoreTransaction(ctx, installation, deps); err != nil {
|
|
return result, err
|
|
}
|
|
state.verified = true
|
|
result.Verified = true
|
|
if state.wasRunning {
|
|
if err := maintenance(ctx, installation, deps.runner, false); err != nil {
|
|
return result, err
|
|
}
|
|
state.maintenanceAttempted = false
|
|
}
|
|
return result, nil
|
|
}
|
|
|
|
func verifyRestoreTransaction(ctx context.Context, installation config.Installation, deps restoreDependencies) error {
|
|
for _, name := range []string{"health", "doctor", "pi", "workspace"} {
|
|
check := deps.verify[name]
|
|
if check == nil {
|
|
return fmt.Errorf("restore verification %q is unavailable", name)
|
|
}
|
|
if err := check(ctx, installation, deps.runner); err != nil {
|
|
return fmt.Errorf("restore verification %s: %w", name, err)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
type verifiedRecoveryError struct{ err error }
|
|
|
|
func (err *verifiedRecoveryError) Error() string { return err.err.Error() }
|
|
|
|
func (err *verifiedRecoveryError) Unwrap() error { return err.err }
|
|
|
|
func recoveryReachedVerifiedState(err error) bool {
|
|
var verifiedErr *verifiedRecoveryError
|
|
return errors.As(err, &verifiedErr)
|
|
}
|
|
|
|
func restoreVerifiedEntries(
|
|
ctx context.Context,
|
|
installation config.Installation,
|
|
preflight PreflightResult,
|
|
archive io.ReaderAt,
|
|
restoreFile func(context.Context, config.Installation, ArchiveEntryMetadata, io.Reader) error,
|
|
restoreVolume func(context.Context, config.Installation, VolumeMetadata, io.Reader) error,
|
|
) error {
|
|
reader, err := zip.NewReader(archive, preflight.ArchiveSize)
|
|
if err != nil {
|
|
return fmt.Errorf("read verified restore archive: %w", err)
|
|
}
|
|
members := make(map[string]*zip.File, len(reader.File))
|
|
for _, member := range reader.File {
|
|
members[member.Name] = member
|
|
}
|
|
for _, entry := range preflight.Entries {
|
|
member := members[entry.Path]
|
|
if member == nil {
|
|
return fmt.Errorf("verified archive is missing %q", entry.Path)
|
|
}
|
|
stream, openErr := member.Open()
|
|
if openErr != nil {
|
|
return fmt.Errorf("open verified archive member %q: %w", entry.Path, openErr)
|
|
}
|
|
var restoreErr error
|
|
if entry.Kind == EntryVolume {
|
|
volume, found := restoreVolumeMetadata(preflight.Manifest, entry.LogicalName)
|
|
if !found {
|
|
_ = stream.Close()
|
|
return errors.New("verified volume metadata is incomplete")
|
|
}
|
|
restoreErr = restoreVolume(ctx, installation, volume, stream)
|
|
} else {
|
|
restoreErr = restoreFile(ctx, installation, entry, stream)
|
|
}
|
|
closeErr := stream.Close()
|
|
if restoreErr != nil {
|
|
return restoreErr
|
|
}
|
|
if closeErr != nil {
|
|
return closeErr
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func restoreVolumeMetadata(manifest Manifest, logicalName string) (VolumeMetadata, bool) {
|
|
if logicalName == "" {
|
|
return VolumeMetadata{}, false
|
|
}
|
|
for _, volume := range manifest.Volumes {
|
|
if volume.LogicalName == logicalName {
|
|
return volume, true
|
|
}
|
|
}
|
|
return VolumeMetadata{}, false
|
|
}
|
|
|
|
// resetAuthenticationState clears browser sessions and pending OIDC transactions without touching
|
|
// installation-global auth.yaml or users.yaml. A root-scoped one-shot repairs ownership and mode
|
|
// before clearing children; links and malformed roots are rejected before any recursive removal.
|
|
func resetAuthenticationState(ctx context.Context, installation config.Installation, runner archiveRunner) error {
|
|
result, err := runner.Run(ctx, installation.ComposeArgs(
|
|
"run", "--rm", "--no-deps", "--no-TTY", "--user", "0:0", "--entrypoint", "sh", "core", "-ceu",
|
|
"test ! -L /data/auth && { test ! -e /data/auth || test -d /data/auth; } && install -d -o 10001 -g 10001 -m 0700 /data/auth && find /data/auth -mindepth 1 -maxdepth 1 -exec rm -rf -- {} + && install -d -o 10001 -g 10001 -m 0700 /data/auth/sessions /data/auth/oidc && chmod 0700 /data/auth /data/auth/sessions /data/auth/oidc && chown 10001:10001 /data/auth /data/auth/sessions /data/auth/oidc && test -z \"$(find /data/auth/sessions /data/auth/oidc -mindepth 1 -print -quit)\"",
|
|
), nil)
|
|
if err != nil {
|
|
return dockerError("reset authentication state", result, err)
|
|
}
|
|
if result.ExitCode != 0 {
|
|
return dockerError("reset authentication state", result, errors.New("Compose returned a nonzero exit status"))
|
|
}
|
|
return nil
|
|
}
|