353 lines
13 KiB
Go
353 lines
13 KiB
Go
package backup
|
|
|
|
import (
|
|
"context"
|
|
"crypto/rand"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"path/filepath"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/aritmolab/thothii/tools/tht/internal/compose"
|
|
"github.com/aritmolab/thothii/tools/tht/internal/config"
|
|
"github.com/aritmolab/thothii/tools/tht/internal/doctor"
|
|
"github.com/aritmolab/thothii/tools/tht/internal/lifecycle"
|
|
"github.com/aritmolab/thothii/tools/tht/internal/pi"
|
|
"github.com/aritmolab/thothii/tools/tht/internal/safeio"
|
|
"github.com/aritmolab/thothii/tools/tht/internal/service"
|
|
)
|
|
|
|
func productionRestoreDependencies(installation config.Installation) restoreDependencies {
|
|
runner := hostRunner{runner: compose.NewRunner(""), binary: "docker", profile: installation.Profile}
|
|
return restoreDependencies{
|
|
preflight: func(ctx context.Context, target config.Installation, request PreflightRequest) (PreflightResult, error) {
|
|
return Preflight(ctx, target, request, PreflightDependencies{
|
|
FreeBytes: restoreFreeBytes,
|
|
CheckOwnershipPermissions: validateRestoreTargets,
|
|
CheckVolumeMapping: func(ctx context.Context, target config.Installation, manifest Manifest) error {
|
|
return validateRestoreVolumes(ctx, target, manifest, runner)
|
|
},
|
|
CheckImageConfigCompatibility: func(ctx context.Context, target config.Installation, manifest Manifest) error {
|
|
return validateRestoreImages(ctx, target, manifest, runner)
|
|
},
|
|
})
|
|
},
|
|
checkpoint: func(ctx context.Context, target config.Installation, request CreateRequest) (Result, error) {
|
|
path, err := restoreCheckpointPath(target, time.Now().UTC())
|
|
if err != nil {
|
|
return Result{}, err
|
|
}
|
|
request.Output = path
|
|
return Create(ctx, target, request)
|
|
},
|
|
acquireLock: func(target config.Installation) (restoreLock, error) {
|
|
return lifecycle.Acquire(target)
|
|
},
|
|
runner: runner,
|
|
sleep: time.Sleep,
|
|
restoreFile: restoreFilePayload,
|
|
restoreVolume: func(ctx context.Context, _ config.Installation, volume VolumeMetadata, input io.Reader) error {
|
|
result, err := runner.Stream(ctx, volumeRestoreCommand(volume.Name), input, io.Discard)
|
|
if err != nil || result.ExitCode != 0 {
|
|
if err == nil {
|
|
err = errors.New("Docker volume helper returned a nonzero exit status")
|
|
}
|
|
return fmt.Errorf("restore volume %s: %w", volume.LogicalName, dockerError("stream volume", result, err))
|
|
}
|
|
return nil
|
|
},
|
|
resetAuthenticationState: resetAuthenticationState,
|
|
verify: map[string]restoreVerify{
|
|
"health": verifyRestoreHealth,
|
|
"doctor": verifyRestoreDoctor,
|
|
"pi": verifyRestorePi,
|
|
"workspace": verifyRestoreWorkspace,
|
|
},
|
|
}
|
|
}
|
|
|
|
func restoreCheckpointPath(installation config.Installation, now time.Time) (string, error) {
|
|
suffix := make([]byte, 8)
|
|
if _, err := rand.Read(suffix); err != nil {
|
|
return "", errors.New("recovery checkpoint name is unavailable")
|
|
}
|
|
name := fmt.Sprintf("restore-checkpoint-%s-%s.zip", now.Format("20060102T150405.000000000Z"), hex.EncodeToString(suffix))
|
|
return filepath.Join(installation.ControlDirectory(), name), nil
|
|
}
|
|
|
|
func validateRestoreTargets(_ context.Context, installation config.Installation, manifest Manifest) error {
|
|
for _, entry := range manifest.Entries {
|
|
if entry.Kind == EntryVolume {
|
|
continue
|
|
}
|
|
if !entry.Archived {
|
|
if entry.Kind == EntrySecretReference || entry.Kind == EntryPreservationReference {
|
|
if err := validateRestoreReference(installation, entry); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
continue
|
|
}
|
|
metadata := ArchiveEntryMetadata{
|
|
Path: entry.Path, Kind: entry.Kind, Owner: entry.Owner, LogicalName: entry.LogicalName,
|
|
SourcePath: entry.SourcePath, Size: entry.Size, SHA256: entry.SHA256, Mode: entry.Mode,
|
|
Sensitive: entry.Sensitive,
|
|
}
|
|
target, err := restoreFileTarget(installation, metadata)
|
|
if err != nil || !safeRestoreParent(target) {
|
|
return errors.New("restore target ownership or permissions are invalid")
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func validateRestoreReference(installation config.Installation, entry Entry) error {
|
|
metadata := ArchiveEntryMetadata{
|
|
Path: entry.Path, Kind: entry.Kind, Owner: entry.Owner, SourcePath: entry.SourcePath,
|
|
Size: entry.Size, SHA256: entry.SHA256, Mode: entry.Mode, Sensitive: entry.Sensitive,
|
|
}
|
|
if entry.Kind == EntryPreservationReference {
|
|
return nil
|
|
}
|
|
if _, err := restoreExternalTarget(installation, metadata); err != nil {
|
|
return errors.New("restore external prerequisite is invalid")
|
|
}
|
|
contents, err := safeio.ReadCanonicalRegular(entry.SourcePath, entry.Size)
|
|
if err != nil || int64(len(contents)) != entry.Size {
|
|
return errors.New("restore external prerequisite is unavailable or unsafe")
|
|
}
|
|
digest := sha256.Sum256(contents)
|
|
if "sha256:"+hex.EncodeToString(digest[:]) != entry.SHA256 {
|
|
return errors.New("restore external prerequisite has changed")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func validateRestoreVolumes(ctx context.Context, installation config.Installation, manifest Manifest, runner archiveRunner) error {
|
|
if len(manifest.Volumes) != len(requiredVolumes) {
|
|
return errors.New("backup volume set is incomplete")
|
|
}
|
|
rendered, err := renderedConfiguration(ctx, installation, runner)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
current, err := inspectRequiredVolumes(ctx, runner, rendered)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
archived := make(map[string]VolumeMetadata, len(manifest.Volumes))
|
|
for _, volume := range manifest.Volumes {
|
|
archived[volume.LogicalName] = volume
|
|
}
|
|
for _, volume := range current {
|
|
previous, found := archived[volume.LogicalName]
|
|
if !found || previous.Name != volume.Name || previous.Driver != volume.Driver {
|
|
return errors.New("backup volume ownership does not match the installation")
|
|
}
|
|
if volume.Labels["com.docker.compose.project"] != installation.ProjectName() {
|
|
return errors.New("current volume is not owned by the installation")
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func validateRestoreImages(ctx context.Context, installation config.Installation, manifest Manifest, runner archiveRunner) error {
|
|
rendered, err := renderedConfiguration(ctx, installation, runner)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, image := range manifest.Images {
|
|
serviceDefinition, found := rendered.Services[image.Service]
|
|
if !found || serviceDefinition.Image == "" || serviceDefinition.Image != image.Reference {
|
|
return errors.New("backup image configuration does not match the installation")
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func restoreFilePayload(_ context.Context, installation config.Installation, entry ArchiveEntryMetadata, input io.Reader) error {
|
|
if entry.Size < 0 || uint64(entry.Size) > defaultPreflightMaxUncompressedBytes {
|
|
return errors.New("restore file size is invalid")
|
|
}
|
|
contents, err := io.ReadAll(io.LimitReader(input, entry.Size+1))
|
|
if err != nil || int64(len(contents)) != entry.Size {
|
|
return errors.New("restore file payload is invalid")
|
|
}
|
|
target, err := restoreFileTarget(installation, entry)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := replaceRestoreFile(target, contents, os.FileMode(entry.Mode)); err != nil {
|
|
return errors.New("restore file could not be replaced safely")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func restoreFileTarget(installation config.Installation, entry ArchiveEntryMetadata) (string, error) {
|
|
if entry.Kind == EntryExternalSecret {
|
|
return restoreExternalTarget(installation, entry)
|
|
}
|
|
if entry.Kind != EntryFile {
|
|
return "", errors.New("restore file kind is unsupported")
|
|
}
|
|
switch entry.Path {
|
|
case "configuration/installation/thothii-installation.yaml":
|
|
return installation.Path, nil
|
|
case "configuration/environment/operator.env":
|
|
return installation.EnvFile, nil
|
|
case "configuration/pi/models.json":
|
|
return filepath.Join(installation.ProjectDirectory, "deploy", "pi", "models.json"), nil
|
|
case "configuration/pi/settings.json":
|
|
return filepath.Join(installation.ProjectDirectory, "deploy", "pi", "settings.json"), nil
|
|
case "configuration/generated/current-image.yaml":
|
|
return installation.CurrentImageOverridePath(), nil
|
|
}
|
|
if strings.HasPrefix(entry.Path, "configuration/overrides/") {
|
|
name := strings.TrimPrefix(entry.Path, "configuration/overrides/")
|
|
indexText, base, found := strings.Cut(name, "-")
|
|
index, parseErr := strconv.Atoi(indexText)
|
|
if !found || parseErr != nil || len(indexText) != 2 || index < 0 || index >= len(installation.Overrides) || filepath.Base(installation.Overrides[index]) != base {
|
|
return "", errors.New("restore override target is invalid")
|
|
}
|
|
return installation.Overrides[index], nil
|
|
}
|
|
if strings.HasPrefix(entry.Owner, "preservation-root:") && strings.HasPrefix(entry.Path, "preservation/") {
|
|
variable := strings.TrimPrefix(entry.Owner, "preservation-root:")
|
|
allowed := variable == "THT_DATA_ROOT" || variable == "THT_PI_STATE_ROOT" || variable == "THT_WORKSPACE_REGISTRY_ROOT"
|
|
parts := strings.SplitN(entry.Path, "/", 3)
|
|
root, rootErr := installation.EnvironmentValue(variable)
|
|
if !allowed || len(parts) != 3 || rootErr != nil || root == "" {
|
|
return "", errors.New("restore preservation target is invalid")
|
|
}
|
|
target := filepath.Join(root, filepath.FromSlash(parts[2]))
|
|
relative, relErr := filepath.Rel(root, target)
|
|
if relErr != nil || relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) {
|
|
return "", errors.New("restore preservation target escapes its root")
|
|
}
|
|
return target, nil
|
|
}
|
|
return "", errors.New("restore file target is not declared")
|
|
}
|
|
|
|
func restoreExternalTarget(installation config.Installation, entry ArchiveEntryMetadata) (string, error) {
|
|
if entry.SourcePath == "" || filepath.Clean(entry.SourcePath) != entry.SourcePath || !filepath.IsAbs(entry.SourcePath) {
|
|
return "", errors.New("restore external target is invalid")
|
|
}
|
|
if entry.Owner == "external-secret" {
|
|
paths, err := installation.SecretFiles()
|
|
if err != nil {
|
|
return "", errors.New("restore external secret declarations are unavailable")
|
|
}
|
|
for _, path := range paths {
|
|
if path == entry.SourcePath {
|
|
return path, nil
|
|
}
|
|
}
|
|
return "", errors.New("restore external secret target is not declared")
|
|
}
|
|
if entry.Owner == "authentication-configuration" {
|
|
for _, name := range []string{"auth.yaml", "users.yaml"} {
|
|
path := filepath.Join(installation.AuthenticationDirectory(), name)
|
|
if entry.SourcePath == path {
|
|
return path, nil
|
|
}
|
|
}
|
|
}
|
|
return "", errors.New("restore external target owner is invalid")
|
|
}
|
|
|
|
func safeRestoreParent(target string) bool {
|
|
if target == "" || !filepath.IsAbs(target) || filepath.Clean(target) != target {
|
|
return false
|
|
}
|
|
parent := filepath.Dir(target)
|
|
resolved, err := filepath.EvalSymlinks(parent)
|
|
if err != nil || resolved != parent {
|
|
return false
|
|
}
|
|
info, err := os.Stat(parent)
|
|
if err != nil || !info.IsDir() {
|
|
return false
|
|
}
|
|
if targetInfo, err := os.Lstat(target); err == nil {
|
|
return targetInfo.Mode().IsRegular() && targetInfo.Mode()&os.ModeSymlink == 0
|
|
} else {
|
|
return errors.Is(err, os.ErrNotExist)
|
|
}
|
|
}
|
|
|
|
func volumeRestoreCommand(volume string) []string {
|
|
return []string{
|
|
"run", "--rm", "--network", "none", "--mount", "type=volume,src=" + volume + ",dst=/target",
|
|
helperImage, "sh", "-ceu",
|
|
"rm -rf -- /target/* /target/.[!.]* /target/..?*; tar --numeric-owner -C /target -xf -",
|
|
}
|
|
}
|
|
|
|
func restoreVerificationRunning(ctx context.Context, installation config.Installation, runner archiveRunner) (bool, error) {
|
|
return installationRunning(ctx, installation, runner)
|
|
}
|
|
|
|
func verifyRestoreHealth(ctx context.Context, installation config.Installation, runner archiveRunner) error {
|
|
running, err := restoreVerificationRunning(ctx, installation, runner)
|
|
if err != nil || !running {
|
|
return err
|
|
}
|
|
return service.WaitForHealthy(ctx, installation, runner)
|
|
}
|
|
|
|
func verifyRestoreDoctor(ctx context.Context, installation config.Installation, runner archiveRunner) error {
|
|
running, err := restoreVerificationRunning(ctx, installation, runner)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !running {
|
|
result, configErr := runner.Run(ctx, installation.ComposeArgs("config", "--quiet"), nil)
|
|
if configErr != nil {
|
|
return dockerError("verify restored Compose configuration", result, configErr)
|
|
}
|
|
return nil
|
|
}
|
|
report, err := doctor.Run(ctx, installation, runner)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !report.OK {
|
|
return errors.New("aggregate doctor did not pass after restore")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func verifyRestorePi(ctx context.Context, installation config.Installation, runner archiveRunner) error {
|
|
running, err := restoreVerificationRunning(ctx, installation, runner)
|
|
if err != nil || !running {
|
|
return err
|
|
}
|
|
return pi.Doctor(ctx, compose.InstallationRunner{Installation: installation, Runner: runner})
|
|
}
|
|
|
|
func verifyRestoreWorkspace(ctx context.Context, installation config.Installation, runner archiveRunner) error {
|
|
running, err := restoreVerificationRunning(ctx, installation, runner)
|
|
if err != nil || !running {
|
|
return err
|
|
}
|
|
result, err := runner.Run(ctx, installation.ComposeArgs(
|
|
"exec", "-T", "core", "curl", "-fsS", "--max-time", "5", "http://127.0.0.1:8787/workspaces",
|
|
), nil)
|
|
if err != nil {
|
|
return dockerError("inspect restored workspaces", result, err)
|
|
}
|
|
var workspaces []json.RawMessage
|
|
if json.Unmarshal([]byte(result.Stdout), &workspaces) != nil {
|
|
return errors.New("restored workspace inspection returned invalid JSON")
|
|
}
|
|
return nil
|
|
}
|