package backup import ( "archive/zip" "context" "errors" "fmt" "io" "time" "github.com/aritmolab/thothii/tools/tht/internal/config" ) var ErrRestoreConfirmationRequired = errors.New("restore requires --yes") // RestoreRequest describes the deliberately-confirmed archive restoration. type RestoreRequest struct { Archive string Confirm bool Drain bool } // RestoreResult records the final service state. Secret-bearing recovery checkpoints are always // destroyed internally and are therefore never exposed to callers. type RestoreResult struct { // Deprecated: always empty. Recovery checkpoints are private transaction internals. Checkpoint string Restarted bool Verified bool } type restoreLock interface{ Release() error } type restoreVerify func(context.Context, config.Installation, archiveRunner) error type restoreDependencies struct { preflight func(context.Context, config.Installation, PreflightRequest) (PreflightResult, error) // checkpointLocked creates the secret-aware recovery archive while the caller already owns // the installation lifecycle lock. It must not call public Create, which would re-acquire the // non-reentrant lock and deadlock the restore transaction. checkpointLocked func(context.Context, config.Installation, CreateRequest) (Result, error) prepareRecovery func(context.Context, config.Installation, string) (PreflightResult, error) recover func(context.Context, config.Installation, PreflightResult, bool) error cleanupCheckpoint func(string) error acquireLock func(config.Installation) (restoreLock, error) runner archiveRunner sleep func(duration time.Duration) restoreFile func(context.Context, config.Installation, ArchiveEntryMetadata, io.Reader) error restoreVolume func(context.Context, config.Installation, VolumeMetadata, io.Reader) error resetAuthenticationState func(context.Context, config.Installation, archiveRunner) error verify map[string]restoreVerify } // restoreTransactionState is the restore admission state machine. The durable maintenance // barrier is released only after the target archive has been verified, or after a separately // verified checkpoint recovery. Every mutable Docker command is tracked before it is invoked. type restoreTransactionState struct { wasRunning bool maintenanceAttempted bool stopAttempted bool mutated bool verified bool recovered bool } func (state restoreTransactionState) recoveryRequired(resultErr error) bool { return resultErr != nil && state.mutated && !state.verified } func (state restoreTransactionState) mayDeactivateMaintenance() bool { return !state.mutated || state.verified || state.recovered } // Restore runs the host transaction through the same concrete Docker/filesystem boundaries used // by backup creation. The injectable core below exists only to make every failure boundary // deterministic in tests. func Restore(ctx context.Context, installation config.Installation, request RestoreRequest) (RestoreResult, error) { return restoreWithDependencies(ctx, installation, request, productionRestoreDependencies(installation)) } func restoreWithDependencies(ctx context.Context, installation config.Installation, request RestoreRequest, deps restoreDependencies) (result RestoreResult, resultErr error) { if !request.Confirm { return RestoreResult{}, ErrRestoreConfirmationRequired } if request.Archive == "" { return RestoreResult{}, errors.New("restore archive is required") } if deps.preflight == nil || deps.checkpointLocked == nil || deps.prepareRecovery == nil || deps.recover == nil || deps.cleanupCheckpoint == nil || deps.acquireLock == nil || deps.runner == nil || deps.restoreFile == nil || deps.restoreVolume == nil || deps.resetAuthenticationState == nil || deps.verify == nil { return RestoreResult{}, errors.New("restore dependencies are incomplete") } preflight, err := deps.preflight(ctx, installation, PreflightRequest{Archive: request.Archive, Confirm: true, AllowExternalSecrets: true}) if err != nil { return RestoreResult{}, err } archive, err := preflight.RevalidateArchive() if err != nil { _ = preflight.CloseArchive() return RestoreResult{}, err } // Lock ordering is lifecycle lock -> Compose/operator maintenance barrier. The core operator // command never acquires the host lifecycle lock, so this order cannot form a lock cycle with // Docker Compose or the durable maintenance marker. lock, err := deps.acquireLock(installation) if err != nil { _ = preflight.CloseArchive() return result, err } defer func() { if releaseErr := lock.Release(); releaseErr != nil { result = RestoreResult{} resultErr = errors.Join(resultErr, fmt.Errorf("release restore lifecycle lock: %w", releaseErr)) } }() defer preflight.CloseArchive() checkpoint, err := deps.checkpointLocked(ctx, installation, CreateRequest{IncludeSecrets: true, Confirm: true}) if err != nil { return result, fmt.Errorf("create recovery checkpoint: %w", err) } recovery, err := deps.prepareRecovery(ctx, installation, checkpoint.Path) if err != nil { cleanupErr := deps.cleanupCheckpoint(checkpoint.Path) return result, errors.Join(fmt.Errorf("validate recovery checkpoint: %w", err), cleanupErr) } defer recovery.CloseArchive() state := restoreTransactionState{} defer func() { if state.recoveryRequired(resultErr) { recoveryContext, cancel := boundedCleanupContext() recoveryErr := deps.recover(recoveryContext, installation, recovery, state.wasRunning) cancel() if recoveryErr != nil { resultErr = errors.Join(resultErr, fmt.Errorf("restore recovery checkpoint: %w", recoveryErr)) if recoveryReachedVerifiedState(recoveryErr) { state.recovered = true state.stopAttempted = false } } else { state.recovered = true state.stopAttempted = false } } checkpointCleanupSucceeded := true var cleanupErr error if checkpointErr := deps.cleanupCheckpoint(checkpoint.Path); checkpointErr != nil { checkpointCleanupSucceeded = false cleanupErr = errors.Join(cleanupErr, fmt.Errorf("destroy recovery checkpoint: %w", checkpointErr)) } if !state.maintenanceAttempted { if cleanupErr != nil { result = RestoreResult{} resultErr = errors.Join(resultErr, cleanupErr) } return } // A recovery checkpoint remains secret-bearing transaction state. Do not reopen // admissions until it has been safely deleted, even if the restored target verified. if !checkpointCleanupSucceeded { result = RestoreResult{} resultErr = errors.Join(resultErr, cleanupErr) return } restartCompleted := true if state.wasRunning && state.stopAttempted && state.mayDeactivateMaintenance() { startErr, started := retryBoundedCleanup(func(cleanupContext context.Context) error { return composeStartAndVerify(cleanupContext, installation, deps.runner) }) if startErr != nil { cleanupErr = errors.Join(cleanupErr, fmt.Errorf("restore maintenance cleanup restart: %w", startErr)) } if started { state.stopAttempted = false } else { restartCompleted = false } } // A failed checkpoint recovery deliberately leaves admissions blocked. Starting or // deactivating at that point would expose an unverified, possibly partial restore. if state.mayDeactivateMaintenance() && restartCompleted { deactivateErr, deactivated := retryBoundedCleanup(func(cleanupContext context.Context) error { return maintenance(cleanupContext, installation, deps.runner, false) }) if deactivateErr != nil { cleanupErr = errors.Join(cleanupErr, fmt.Errorf("restore maintenance cleanup: %w", deactivateErr)) } if deactivated { state.maintenanceAttempted = false } } if cleanupErr != nil { result = RestoreResult{} resultErr = errors.Join(resultErr, cleanupErr) } }() wasRunning, err := installationRunning(ctx, installation, deps.runner) if err != nil { return result, err } state.wasRunning = wasRunning if state.wasRunning { // The activation command may take effect even when its response is lost. Track the attempt, // not merely a successful return, so every subsequent path compensates from durable state. state.maintenanceAttempted = true if err := maintenance(ctx, installation, deps.runner, true); err != nil { return result, err } if err := waitForNoActiveSessions(ctx, installation, deps.runner, request.Drain, deps.sleep); err != nil { return result, err } // Compose may stop the core and then lose its response. Cleanup must therefore restart after // any stop attempt, including a command that returns an error. state.stopAttempted = true if err := runCompose(ctx, installation, deps.runner, "stop"); err != nil { return result, err } } state.mutated = true if err := restoreVerifiedEntries(ctx, installation, preflight, archive, deps.restoreFile, deps.restoreVolume); err != nil { return result, err } if err := deps.resetAuthenticationState(ctx, installation, deps.runner); err != nil { return result, fmt.Errorf("reset authentication state: %w", err) } if state.wasRunning { if err := composeStartAndVerify(ctx, installation, deps.runner); err != nil { return result, err } state.stopAttempted = false result.Restarted = true } if err := verifyRestoreTransaction(ctx, installation, deps); err != nil { return result, err } state.verified = true result.Verified = true return result, nil } func verifyRestoreTransaction(ctx context.Context, installation config.Installation, deps restoreDependencies) error { for _, name := range []string{"health", "doctor", "pi", "workspace"} { check := deps.verify[name] if check == nil { return fmt.Errorf("restore verification %q is unavailable", name) } if err := check(ctx, installation, deps.runner); err != nil { return fmt.Errorf("restore verification %s: %w", name, err) } } return nil } type verifiedRecoveryError struct{ err error } func (err *verifiedRecoveryError) Error() string { return err.err.Error() } func (err *verifiedRecoveryError) Unwrap() error { return err.err } func recoveryReachedVerifiedState(err error) bool { var verifiedErr *verifiedRecoveryError return errors.As(err, &verifiedErr) } func restoreVerifiedEntries( ctx context.Context, installation config.Installation, preflight PreflightResult, archive io.ReaderAt, restoreFile func(context.Context, config.Installation, ArchiveEntryMetadata, io.Reader) error, restoreVolume func(context.Context, config.Installation, VolumeMetadata, io.Reader) error, ) error { reader, err := zip.NewReader(archive, preflight.ArchiveSize) if err != nil { return fmt.Errorf("read verified restore archive: %w", err) } members := make(map[string]*zip.File, len(reader.File)) for _, member := range reader.File { members[member.Name] = member } for _, entry := range preflight.Entries { member := members[entry.Path] if member == nil { return fmt.Errorf("verified archive is missing %q", entry.Path) } stream, openErr := member.Open() if openErr != nil { return fmt.Errorf("open verified archive member %q: %w", entry.Path, openErr) } var restoreErr error if entry.Kind == EntryVolume { volume, found := restoreVolumeMetadata(preflight.Manifest, entry.LogicalName) if !found { _ = stream.Close() return errors.New("verified volume metadata is incomplete") } restoreErr = restoreVolume(ctx, installation, volume, stream) } else { restoreErr = restoreFile(ctx, installation, entry, stream) } closeErr := stream.Close() if restoreErr != nil { return restoreErr } if closeErr != nil { return closeErr } } return nil } func restoreVolumeMetadata(manifest Manifest, logicalName string) (VolumeMetadata, bool) { if logicalName == "" { return VolumeMetadata{}, false } for _, volume := range manifest.Volumes { if volume.LogicalName == logicalName { return volume, true } } return VolumeMetadata{}, false } // resetAuthenticationState clears browser sessions and pending OIDC transactions without touching // installation-global auth.yaml or users.yaml. A root-scoped one-shot repairs ownership and mode // before clearing children; links and malformed roots are rejected before any recursive removal. func resetAuthenticationState(ctx context.Context, installation config.Installation, runner archiveRunner) error { result, err := runner.Run(ctx, installation.ComposeArgs( "run", "--rm", "--no-deps", "--no-TTY", "--user", "0:0", "--entrypoint", "sh", "core", "-ceu", "test ! -L /data/auth && { test ! -e /data/auth || test -d /data/auth; } && install -d -o 10001 -g 10001 -m 0700 /data/auth && find /data/auth -mindepth 1 -maxdepth 1 -exec rm -rf -- {} + && install -d -o 10001 -g 10001 -m 0700 /data/auth/sessions /data/auth/oidc && chmod 0700 /data/auth /data/auth/sessions /data/auth/oidc && chown 10001:10001 /data/auth /data/auth/sessions /data/auth/oidc && test -z \"$(find /data/auth/sessions /data/auth/oidc -mindepth 1 -print -quit)\"", ), nil) if err != nil { return dockerError("reset authentication state", result, err) } if result.ExitCode != 0 { return dockerError("reset authentication state", result, errors.New("Compose returned a nonzero exit status")) } return nil }