fix(auth): address Task 13 deployment review findings
This commit is contained in:
@@ -457,26 +457,6 @@ func inspectRequiredVolumes(ctx context.Context, runner archiveRunner, rendered
|
||||
}
|
||||
|
||||
func imageIdentities(ctx context.Context, installation config.Installation, runner archiveRunner, rendered renderedCompose) ([]ImageIdentity, error) {
|
||||
result, err := runner.Run(ctx, installation.ComposeArgs("images", "--format", "json"), nil)
|
||||
if err != nil {
|
||||
return nil, dockerError("inspect image identities", result, err)
|
||||
}
|
||||
ids := map[string]string{}
|
||||
decoder := json.NewDecoder(strings.NewReader(result.Stdout))
|
||||
for {
|
||||
var item struct {
|
||||
Service string `json:"Service"`
|
||||
ID string `json:"ID"`
|
||||
}
|
||||
err := decoder.Decode(&item)
|
||||
if errors.Is(err, io.EOF) {
|
||||
break
|
||||
}
|
||||
if err != nil || item.Service == "" {
|
||||
return nil, errors.New("Docker Compose returned invalid image identities")
|
||||
}
|
||||
ids[item.Service] = item.ID
|
||||
}
|
||||
services := make([]string, 0, len(rendered.Services))
|
||||
for name, definition := range rendered.Services {
|
||||
if definition.Image != "" {
|
||||
@@ -486,11 +466,70 @@ func imageIdentities(ctx context.Context, installation config.Installation, runn
|
||||
sort.Strings(services)
|
||||
images := make([]ImageIdentity, 0, len(services))
|
||||
for _, name := range services {
|
||||
images = append(images, ImageIdentity{Service: name, Reference: rendered.Services[name].Image, ID: ids[name]})
|
||||
result, err := runner.Run(ctx, installation.ComposeArgs("images", "--format", "json", name), nil)
|
||||
if err != nil {
|
||||
return nil, dockerError("inspect image identities", result, err)
|
||||
}
|
||||
inspected, err := decodeComposeImageIdentities(result.Stdout)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
matching := make([]composeImageIdentity, 0, len(inspected))
|
||||
for _, item := range inspected {
|
||||
if item.Service == "" || item.Service == name {
|
||||
matching = append(matching, item)
|
||||
}
|
||||
}
|
||||
if len(matching) > 1 {
|
||||
return nil, errors.New("Docker Compose returned invalid image identities")
|
||||
}
|
||||
id := ""
|
||||
if len(matching) == 1 {
|
||||
id = matching[0].ID
|
||||
}
|
||||
images = append(images, ImageIdentity{Service: name, Reference: rendered.Services[name].Image, ID: id})
|
||||
}
|
||||
return images, nil
|
||||
}
|
||||
|
||||
type composeImageIdentity struct {
|
||||
Service string `json:"Service"`
|
||||
ContainerName string `json:"ContainerName"`
|
||||
ID string `json:"ID"`
|
||||
}
|
||||
|
||||
func decodeComposeImageIdentities(value string) ([]composeImageIdentity, error) {
|
||||
trimmed := strings.TrimSpace(value)
|
||||
if trimmed == "" {
|
||||
return nil, nil
|
||||
}
|
||||
var identities []composeImageIdentity
|
||||
if strings.HasPrefix(trimmed, "[") {
|
||||
if json.Unmarshal([]byte(trimmed), &identities) != nil {
|
||||
return nil, errors.New("Docker Compose returned invalid image identities")
|
||||
}
|
||||
} else {
|
||||
decoder := json.NewDecoder(strings.NewReader(trimmed))
|
||||
for {
|
||||
var item composeImageIdentity
|
||||
err := decoder.Decode(&item)
|
||||
if errors.Is(err, io.EOF) {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return nil, errors.New("Docker Compose returned invalid image identities")
|
||||
}
|
||||
identities = append(identities, item)
|
||||
}
|
||||
}
|
||||
for _, item := range identities {
|
||||
if item.ID == "" || item.Service == "" && item.ContainerName == "" {
|
||||
return nil, errors.New("Docker Compose returned invalid image identities")
|
||||
}
|
||||
}
|
||||
return identities, nil
|
||||
}
|
||||
|
||||
func installationRunning(ctx context.Context, installation config.Installation, runner archiveRunner) (bool, error) {
|
||||
result, err := runner.Run(ctx, installation.ComposeArgs("ps", "--all", "--format", "json"), nil)
|
||||
if err != nil {
|
||||
|
||||
@@ -23,6 +23,32 @@ import (
|
||||
|
||||
var requiredTestVolumes = []string{"settings", "pi-state", "workspace-registry", "workspace-secrets", "sessions", "qdrant-data", "embedding-models"}
|
||||
|
||||
func TestDecodeComposeImageIdentitiesAcceptsArrayAndStreamingJSON(t *testing.T) {
|
||||
for name, input := range map[string]string{
|
||||
"array": `[{"ContainerName":"project-core-1","ID":"sha256:core"},{"ContainerName":"project-frontend-1","ID":"sha256:frontend"}]`,
|
||||
"streaming": "{\"Service\":\"core\",\"ID\":\"sha256:core\"}\n{\"Service\":\"frontend\",\"ID\":\"sha256:frontend\"}\n",
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
identities, err := decodeComposeImageIdentities(input)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(identities) != 2 || identities[0].ID != "sha256:core" || identities[1].ID != "sha256:frontend" {
|
||||
t.Fatalf("image identities = %#v", identities)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeComposeImageIdentitiesAcceptsNoContainerForProfiledService(t *testing.T) {
|
||||
for _, input := range []string{"", "[]"} {
|
||||
identities, err := decodeComposeImageIdentities(input)
|
||||
if err != nil || len(identities) != 0 {
|
||||
t.Fatalf("decodeComposeImageIdentities(%q) = %#v, %v", input, identities, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateWritesManifestLastWithConfigurationMetadataAndSevenVolumes(t *testing.T) {
|
||||
fixture := newBackupFixture(t, "local")
|
||||
output := filepath.Join(t.TempDir(), "custom.zip")
|
||||
|
||||
@@ -61,13 +61,15 @@ type PreflightDependencies struct {
|
||||
|
||||
// ArchiveEntryMetadata is safe restore metadata. It intentionally contains no archive payload.
|
||||
type ArchiveEntryMetadata struct {
|
||||
Path string
|
||||
Kind string
|
||||
Owner string
|
||||
Size int64
|
||||
SHA256 string
|
||||
Mode uint32
|
||||
Sensitive bool
|
||||
Path string
|
||||
Kind string
|
||||
Owner string
|
||||
LogicalName string
|
||||
SourcePath string
|
||||
Size int64
|
||||
SHA256 string
|
||||
Mode uint32
|
||||
Sensitive bool
|
||||
}
|
||||
|
||||
// PreflightResult is the validated, non-mutating input for a future restore transaction.
|
||||
@@ -534,6 +536,8 @@ func reconcileArchiveEntries(ctx context.Context, manifest Manifest, entries map
|
||||
requiredBytes += uint64(actual.metadata.Size)
|
||||
actual.metadata.Kind = entry.Kind
|
||||
actual.metadata.Owner = entry.Owner
|
||||
actual.metadata.LogicalName = entry.LogicalName
|
||||
actual.metadata.SourcePath = entry.SourcePath
|
||||
actual.metadata.Sensitive = entry.Sensitive
|
||||
metadata = append(metadata, actual.metadata)
|
||||
delete(entries, entry.Path)
|
||||
|
||||
@@ -325,6 +325,7 @@ type preflightArchiveSpec struct {
|
||||
entries []preflightArchiveEntry
|
||||
rawEntries []preflightRawArchiveEntry
|
||||
rawManifest []byte
|
||||
volumes []VolumeMetadata
|
||||
}
|
||||
|
||||
type preflightArchiveEntry struct {
|
||||
@@ -337,6 +338,9 @@ type preflightArchiveEntry struct {
|
||||
mode os.FileMode
|
||||
manifestMode *uint32
|
||||
method uint16
|
||||
owner string
|
||||
logicalName string
|
||||
sourcePath string
|
||||
}
|
||||
|
||||
type preflightRawArchiveEntry struct {
|
||||
@@ -374,6 +378,7 @@ func writePreflightArchive(t *testing.T, archivePath string, spec preflightArchi
|
||||
SchemaVersion: spec.schemaVersion, InstallationID: spec.installationID,
|
||||
CreatedAt: time.Date(2026, 8, 16, 10, 0, 0, 0, time.UTC), SourceRevision: testRevision,
|
||||
IncludesSecrets: spec.includeSecrets, ComposeProject: "thothii-test",
|
||||
Volumes: append([]VolumeMetadata(nil), spec.volumes...),
|
||||
}
|
||||
for _, entry := range spec.entries {
|
||||
checksum := entry.checksum
|
||||
@@ -384,11 +389,18 @@ func writePreflightArchive(t *testing.T, archivePath string, spec preflightArchi
|
||||
if kind == "" {
|
||||
kind = EntryFile
|
||||
}
|
||||
sourcePath := ""
|
||||
owner := "installation"
|
||||
sourcePath := entry.sourcePath
|
||||
owner := entry.owner
|
||||
if owner == "" {
|
||||
owner = "installation"
|
||||
}
|
||||
if kind == EntryExternalSecret {
|
||||
sourcePath = "/protected/secret"
|
||||
owner = "external-secret"
|
||||
if sourcePath == "" {
|
||||
sourcePath = "/protected/secret"
|
||||
}
|
||||
if entry.owner == "" {
|
||||
owner = "external-secret"
|
||||
}
|
||||
}
|
||||
mode := uint32(entry.mode.Perm())
|
||||
if mode == 0 {
|
||||
@@ -397,7 +409,7 @@ func writePreflightArchive(t *testing.T, archivePath string, spec preflightArchi
|
||||
if entry.manifestMode != nil {
|
||||
mode = *entry.manifestMode
|
||||
}
|
||||
manifest.Entries = append(manifest.Entries, Entry{Path: entry.path, Kind: kind, Owner: owner, SourcePath: sourcePath, SHA256: checksum, Size: int64(len(entry.body)), Mode: mode, Archived: true, Sensitive: entry.sensitive})
|
||||
manifest.Entries = append(manifest.Entries, Entry{Path: entry.path, Kind: kind, Owner: owner, LogicalName: entry.logicalName, SourcePath: sourcePath, SHA256: checksum, Size: int64(len(entry.body)), Mode: mode, Archived: true, Sensitive: entry.sensitive})
|
||||
}
|
||||
manifestBytes, err := manifest.JSON()
|
||||
if err != nil {
|
||||
|
||||
@@ -42,10 +42,11 @@ type restoreDependencies struct {
|
||||
verify map[string]restoreVerify
|
||||
}
|
||||
|
||||
// Restore runs the host transaction. Concrete host dependencies are intentionally kept outside
|
||||
// the deterministic core so callers cannot bypass its preflight and checkpoint boundaries.
|
||||
// Restore runs the host transaction through the same concrete Docker/filesystem boundaries used
|
||||
// by backup creation. The injectable core below exists only to make every failure boundary
|
||||
// deterministic in tests.
|
||||
func Restore(ctx context.Context, installation config.Installation, request RestoreRequest) (RestoreResult, error) {
|
||||
return RestoreResult{}, errors.New("restore host dependencies are unavailable")
|
||||
return restoreWithDependencies(ctx, installation, request, productionRestoreDependencies(installation))
|
||||
}
|
||||
|
||||
func restoreWithDependencies(ctx context.Context, installation config.Installation, request RestoreRequest, deps restoreDependencies) (result RestoreResult, resultErr error) {
|
||||
@@ -113,9 +114,6 @@ func restoreWithDependencies(ctx context.Context, installation config.Installati
|
||||
members[member.Name] = member
|
||||
}
|
||||
for _, entry := range preflight.Entries {
|
||||
if entry.Kind == EntryVolume {
|
||||
continue
|
||||
}
|
||||
member := members[entry.Path]
|
||||
if member == nil {
|
||||
return result, fmt.Errorf("verified archive is missing %q", entry.Path)
|
||||
@@ -125,7 +123,17 @@ func restoreWithDependencies(ctx context.Context, installation config.Installati
|
||||
return result, fmt.Errorf("open verified archive member %q: %w", entry.Path, openErr)
|
||||
}
|
||||
mutated = true
|
||||
restoreErr := deps.restoreFile(ctx, installation, entry, stream)
|
||||
var restoreErr error
|
||||
if entry.Kind == EntryVolume {
|
||||
volume, found := restoreVolumeMetadata(preflight.Manifest, entry.LogicalName)
|
||||
if !found {
|
||||
_ = stream.Close()
|
||||
return result, errors.New("verified volume metadata is incomplete")
|
||||
}
|
||||
restoreErr = deps.restoreVolume(ctx, installation, volume, stream)
|
||||
} else {
|
||||
restoreErr = deps.restoreFile(ctx, installation, entry, stream)
|
||||
}
|
||||
closeErr := stream.Close()
|
||||
if restoreErr != nil {
|
||||
return result, restoreErr
|
||||
@@ -156,13 +164,25 @@ func restoreWithDependencies(ctx context.Context, installation config.Installati
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func restoreVolumeMetadata(manifest Manifest, logicalName string) (VolumeMetadata, bool) {
|
||||
if logicalName == "" {
|
||||
return VolumeMetadata{}, false
|
||||
}
|
||||
for _, volume := range manifest.Volumes {
|
||||
if volume.LogicalName == logicalName {
|
||||
return volume, true
|
||||
}
|
||||
}
|
||||
return VolumeMetadata{}, false
|
||||
}
|
||||
|
||||
// resetAuthenticationState clears browser sessions and pending OIDC transactions without touching
|
||||
// installation-global auth.yaml or users.yaml. The command runs as the unprivileged core user so
|
||||
// the recreated state root is private to the service on both the local volume and server /data bind.
|
||||
func resetAuthenticationState(ctx context.Context, installation config.Installation, runner archiveRunner) error {
|
||||
result, err := runner.Run(ctx, installation.ComposeArgs(
|
||||
"run", "--rm", "--no-deps", "--no-TTY", "--entrypoint", "sh", "core", "-ceu",
|
||||
"rm -rf /data/auth && install -d -m 0700 /data/auth /data/auth/sessions /data/auth/oidc",
|
||||
"find /data/auth -mindepth 1 -maxdepth 1 -exec rm -rf -- {} + && install -d -m 0700 /data/auth /data/auth/sessions /data/auth/oidc && test -z \"$(find /data/auth/sessions /data/auth/oidc -mindepth 1 -print -quit)\"",
|
||||
), nil)
|
||||
if err != nil {
|
||||
return dockerError("reset authentication state", result, err)
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
//go:build !windows
|
||||
|
||||
package backup
|
||||
|
||||
import (
|
||||
"errors"
|
||||
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
func restoreFreeBytes(target string) (uint64, error) {
|
||||
var statistics unix.Statfs_t
|
||||
if err := unix.Statfs(target, &statistics); err != nil {
|
||||
return 0, errors.New("restore filesystem capacity is unavailable")
|
||||
}
|
||||
blockSize := uint64(statistics.Bsize)
|
||||
available := uint64(statistics.Bavail)
|
||||
if blockSize != 0 && available > ^uint64(0)/blockSize {
|
||||
return 0, errors.New("restore filesystem capacity is invalid")
|
||||
}
|
||||
return blockSize * available, nil
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
//go:build windows
|
||||
|
||||
package backup
|
||||
|
||||
import (
|
||||
"errors"
|
||||
|
||||
"golang.org/x/sys/windows"
|
||||
)
|
||||
|
||||
func restoreFreeBytes(target string) (uint64, error) {
|
||||
path, err := windows.UTF16PtrFromString(target)
|
||||
if err != nil {
|
||||
return 0, errors.New("restore filesystem capacity is unavailable")
|
||||
}
|
||||
var available uint64
|
||||
if err := windows.GetDiskFreeSpaceEx(path, &available, nil, nil); err != nil {
|
||||
return 0, errors.New("restore filesystem capacity is unavailable")
|
||||
}
|
||||
return available, nil
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
//go:build !windows
|
||||
|
||||
package backup
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/aritmolab/thothii/tools/tht/internal/safeio"
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
type restoreTargetIdentity struct {
|
||||
exists bool
|
||||
device uint64
|
||||
inode uint64
|
||||
}
|
||||
|
||||
func replaceRestoreFile(target string, contents []byte, mode os.FileMode) error {
|
||||
if safeio.ValidateCanonicalPath(target) != nil || mode&os.ModeType != 0 || mode.Perm() == 0 {
|
||||
return safeio.ErrUnsafeFile
|
||||
}
|
||||
components := strings.Split(strings.TrimPrefix(target, string(os.PathSeparator)), string(os.PathSeparator))
|
||||
if len(components) < 2 || components[0] == "" || components[len(components)-1] == "" {
|
||||
return safeio.ErrUnsafeFile
|
||||
}
|
||||
directory, err := unix.Open(string(os.PathSeparator), unix.O_RDONLY|unix.O_CLOEXEC|unix.O_DIRECTORY, 0)
|
||||
if err != nil {
|
||||
return safeio.ErrUnsafeFile
|
||||
}
|
||||
defer unix.Close(directory)
|
||||
for _, component := range components[:len(components)-1] {
|
||||
next, openErr := unix.Openat(directory, component, unix.O_RDONLY|unix.O_CLOEXEC|unix.O_DIRECTORY|unix.O_NOFOLLOW, 0)
|
||||
if openErr != nil {
|
||||
return safeio.ErrUnsafeFile
|
||||
}
|
||||
unix.Close(directory)
|
||||
directory = next
|
||||
}
|
||||
name := components[len(components)-1]
|
||||
identity, err := inspectRestoreTargetAt(directory, name)
|
||||
if err != nil {
|
||||
return safeio.ErrUnsafeFile
|
||||
}
|
||||
temporary, err := writeRestoreTemporaryAt(directory, contents, mode.Perm())
|
||||
if err != nil {
|
||||
return safeio.ErrUnsafeFile
|
||||
}
|
||||
defer func() { _ = unix.Unlinkat(directory, temporary, 0) }()
|
||||
current, err := inspectRestoreTargetAt(directory, name)
|
||||
if err != nil || current != identity {
|
||||
return safeio.ErrUnsafeFile
|
||||
}
|
||||
if err := unix.Renameat(directory, temporary, directory, name); err != nil {
|
||||
return safeio.ErrUnsafeFile
|
||||
}
|
||||
temporary = ""
|
||||
if err := unix.Fsync(directory); err != nil {
|
||||
return safeio.ErrUnsafeFile
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func inspectRestoreTargetAt(directory int, name string) (restoreTargetIdentity, error) {
|
||||
var status unix.Stat_t
|
||||
err := unix.Fstatat(directory, name, &status, unix.AT_SYMLINK_NOFOLLOW)
|
||||
if errors.Is(err, unix.ENOENT) {
|
||||
return restoreTargetIdentity{}, nil
|
||||
}
|
||||
if err != nil || status.Mode&unix.S_IFMT != unix.S_IFREG || status.Nlink != 1 {
|
||||
return restoreTargetIdentity{}, safeio.ErrUnsafeFile
|
||||
}
|
||||
return restoreTargetIdentity{exists: true, device: uint64(status.Dev), inode: status.Ino}, nil
|
||||
}
|
||||
|
||||
func writeRestoreTemporaryAt(directory int, contents []byte, mode os.FileMode) (string, error) {
|
||||
for attempt := 0; attempt < 16; attempt++ {
|
||||
random := make([]byte, 8)
|
||||
if _, err := rand.Read(random); err != nil {
|
||||
return "", err
|
||||
}
|
||||
name := ".tht-restore-" + hex.EncodeToString(random) + ".tmp"
|
||||
descriptor, err := unix.Openat(directory, name, unix.O_WRONLY|unix.O_CREAT|unix.O_EXCL|unix.O_CLOEXEC|unix.O_NOFOLLOW, uint32(mode))
|
||||
if errors.Is(err, unix.EEXIST) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
file := os.NewFile(uintptr(descriptor), filepath.Base(name))
|
||||
if file == nil {
|
||||
unix.Close(descriptor)
|
||||
return "", safeio.ErrUnsafeFile
|
||||
}
|
||||
if err := file.Chmod(mode); err == nil {
|
||||
var written int
|
||||
written, err = file.Write(contents)
|
||||
if err == nil && written != len(contents) {
|
||||
err = io.ErrShortWrite
|
||||
}
|
||||
}
|
||||
if err == nil {
|
||||
err = file.Sync()
|
||||
}
|
||||
closeErr := file.Close()
|
||||
if err == nil {
|
||||
err = closeErr
|
||||
}
|
||||
if err != nil {
|
||||
_ = unix.Unlinkat(directory, name, 0)
|
||||
return "", err
|
||||
}
|
||||
return name, nil
|
||||
}
|
||||
return "", safeio.ErrUnsafeFile
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
//go:build windows
|
||||
|
||||
package backup
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/aritmolab/thothii/tools/tht/internal/safeio"
|
||||
"golang.org/x/sys/windows"
|
||||
)
|
||||
|
||||
func replaceRestoreFile(target string, contents []byte, mode os.FileMode) error {
|
||||
if safeio.ValidateCanonicalPath(target) != nil || mode&os.ModeType != 0 || mode.Perm() == 0 {
|
||||
return safeio.ErrUnsafeFile
|
||||
}
|
||||
parent := filepath.Dir(target)
|
||||
resolved, err := filepath.EvalSymlinks(parent)
|
||||
if err != nil || resolved != parent || !safeWindowsRestoreTarget(target) {
|
||||
return safeio.ErrUnsafeFile
|
||||
}
|
||||
temporary, err := os.CreateTemp(parent, ".tht-restore-*.tmp")
|
||||
if err != nil {
|
||||
return safeio.ErrUnsafeFile
|
||||
}
|
||||
temporaryPath := temporary.Name()
|
||||
defer os.Remove(temporaryPath)
|
||||
if err := temporary.Chmod(mode.Perm()); err == nil {
|
||||
_, err = temporary.Write(contents)
|
||||
}
|
||||
if err == nil {
|
||||
err = temporary.Sync()
|
||||
}
|
||||
closeErr := temporary.Close()
|
||||
if err == nil {
|
||||
err = closeErr
|
||||
}
|
||||
if err != nil || !safeWindowsRestoreTarget(target) {
|
||||
return safeio.ErrUnsafeFile
|
||||
}
|
||||
from, fromErr := windows.UTF16PtrFromString(temporaryPath)
|
||||
to, toErr := windows.UTF16PtrFromString(target)
|
||||
if fromErr != nil || toErr != nil {
|
||||
return safeio.ErrUnsafeFile
|
||||
}
|
||||
if err := windows.MoveFileEx(from, to, windows.MOVEFILE_REPLACE_EXISTING|windows.MOVEFILE_WRITE_THROUGH); err != nil {
|
||||
return safeio.ErrUnsafeFile
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func safeWindowsRestoreTarget(target string) bool {
|
||||
info, err := os.Lstat(target)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return true
|
||||
}
|
||||
return err == nil && info.Mode().IsRegular() && info.Mode()&os.ModeSymlink == 0
|
||||
}
|
||||
@@ -0,0 +1,352 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/aritmolab/thothii/tools/tht/internal/compose"
|
||||
"github.com/aritmolab/thothii/tools/tht/internal/config"
|
||||
"github.com/aritmolab/thothii/tools/tht/internal/doctor"
|
||||
"github.com/aritmolab/thothii/tools/tht/internal/lifecycle"
|
||||
"github.com/aritmolab/thothii/tools/tht/internal/pi"
|
||||
"github.com/aritmolab/thothii/tools/tht/internal/safeio"
|
||||
"github.com/aritmolab/thothii/tools/tht/internal/service"
|
||||
)
|
||||
|
||||
func productionRestoreDependencies(installation config.Installation) restoreDependencies {
|
||||
runner := hostRunner{runner: compose.NewRunner(""), binary: "docker", profile: installation.Profile}
|
||||
return restoreDependencies{
|
||||
preflight: func(ctx context.Context, target config.Installation, request PreflightRequest) (PreflightResult, error) {
|
||||
return Preflight(ctx, target, request, PreflightDependencies{
|
||||
FreeBytes: restoreFreeBytes,
|
||||
CheckOwnershipPermissions: validateRestoreTargets,
|
||||
CheckVolumeMapping: func(ctx context.Context, target config.Installation, manifest Manifest) error {
|
||||
return validateRestoreVolumes(ctx, target, manifest, runner)
|
||||
},
|
||||
CheckImageConfigCompatibility: func(ctx context.Context, target config.Installation, manifest Manifest) error {
|
||||
return validateRestoreImages(ctx, target, manifest, runner)
|
||||
},
|
||||
})
|
||||
},
|
||||
checkpoint: func(ctx context.Context, target config.Installation, request CreateRequest) (Result, error) {
|
||||
path, err := restoreCheckpointPath(target, time.Now().UTC())
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
request.Output = path
|
||||
return Create(ctx, target, request)
|
||||
},
|
||||
acquireLock: func(target config.Installation) (restoreLock, error) {
|
||||
return lifecycle.Acquire(target)
|
||||
},
|
||||
runner: runner,
|
||||
sleep: time.Sleep,
|
||||
restoreFile: restoreFilePayload,
|
||||
restoreVolume: func(ctx context.Context, _ config.Installation, volume VolumeMetadata, input io.Reader) error {
|
||||
result, err := runner.Stream(ctx, volumeRestoreCommand(volume.Name), input, io.Discard)
|
||||
if err != nil || result.ExitCode != 0 {
|
||||
if err == nil {
|
||||
err = errors.New("Docker volume helper returned a nonzero exit status")
|
||||
}
|
||||
return fmt.Errorf("restore volume %s: %w", volume.LogicalName, dockerError("stream volume", result, err))
|
||||
}
|
||||
return nil
|
||||
},
|
||||
resetAuthenticationState: resetAuthenticationState,
|
||||
verify: map[string]restoreVerify{
|
||||
"health": verifyRestoreHealth,
|
||||
"doctor": verifyRestoreDoctor,
|
||||
"pi": verifyRestorePi,
|
||||
"workspace": verifyRestoreWorkspace,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func restoreCheckpointPath(installation config.Installation, now time.Time) (string, error) {
|
||||
suffix := make([]byte, 8)
|
||||
if _, err := rand.Read(suffix); err != nil {
|
||||
return "", errors.New("recovery checkpoint name is unavailable")
|
||||
}
|
||||
name := fmt.Sprintf("restore-checkpoint-%s-%s.zip", now.Format("20060102T150405.000000000Z"), hex.EncodeToString(suffix))
|
||||
return filepath.Join(installation.ControlDirectory(), name), nil
|
||||
}
|
||||
|
||||
func validateRestoreTargets(_ context.Context, installation config.Installation, manifest Manifest) error {
|
||||
for _, entry := range manifest.Entries {
|
||||
if entry.Kind == EntryVolume {
|
||||
continue
|
||||
}
|
||||
if !entry.Archived {
|
||||
if entry.Kind == EntrySecretReference || entry.Kind == EntryPreservationReference {
|
||||
if err := validateRestoreReference(installation, entry); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
metadata := ArchiveEntryMetadata{
|
||||
Path: entry.Path, Kind: entry.Kind, Owner: entry.Owner, LogicalName: entry.LogicalName,
|
||||
SourcePath: entry.SourcePath, Size: entry.Size, SHA256: entry.SHA256, Mode: entry.Mode,
|
||||
Sensitive: entry.Sensitive,
|
||||
}
|
||||
target, err := restoreFileTarget(installation, metadata)
|
||||
if err != nil || !safeRestoreParent(target) {
|
||||
return errors.New("restore target ownership or permissions are invalid")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateRestoreReference(installation config.Installation, entry Entry) error {
|
||||
metadata := ArchiveEntryMetadata{
|
||||
Path: entry.Path, Kind: entry.Kind, Owner: entry.Owner, SourcePath: entry.SourcePath,
|
||||
Size: entry.Size, SHA256: entry.SHA256, Mode: entry.Mode, Sensitive: entry.Sensitive,
|
||||
}
|
||||
if entry.Kind == EntryPreservationReference {
|
||||
return nil
|
||||
}
|
||||
if _, err := restoreExternalTarget(installation, metadata); err != nil {
|
||||
return errors.New("restore external prerequisite is invalid")
|
||||
}
|
||||
contents, err := safeio.ReadCanonicalRegular(entry.SourcePath, entry.Size)
|
||||
if err != nil || int64(len(contents)) != entry.Size {
|
||||
return errors.New("restore external prerequisite is unavailable or unsafe")
|
||||
}
|
||||
digest := sha256.Sum256(contents)
|
||||
if "sha256:"+hex.EncodeToString(digest[:]) != entry.SHA256 {
|
||||
return errors.New("restore external prerequisite has changed")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateRestoreVolumes(ctx context.Context, installation config.Installation, manifest Manifest, runner archiveRunner) error {
|
||||
if len(manifest.Volumes) != len(requiredVolumes) {
|
||||
return errors.New("backup volume set is incomplete")
|
||||
}
|
||||
rendered, err := renderedConfiguration(ctx, installation, runner)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
current, err := inspectRequiredVolumes(ctx, runner, rendered)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
archived := make(map[string]VolumeMetadata, len(manifest.Volumes))
|
||||
for _, volume := range manifest.Volumes {
|
||||
archived[volume.LogicalName] = volume
|
||||
}
|
||||
for _, volume := range current {
|
||||
previous, found := archived[volume.LogicalName]
|
||||
if !found || previous.Name != volume.Name || previous.Driver != volume.Driver {
|
||||
return errors.New("backup volume ownership does not match the installation")
|
||||
}
|
||||
if volume.Labels["com.docker.compose.project"] != installation.ProjectName() {
|
||||
return errors.New("current volume is not owned by the installation")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateRestoreImages(ctx context.Context, installation config.Installation, manifest Manifest, runner archiveRunner) error {
|
||||
rendered, err := renderedConfiguration(ctx, installation, runner)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, image := range manifest.Images {
|
||||
serviceDefinition, found := rendered.Services[image.Service]
|
||||
if !found || serviceDefinition.Image == "" || serviceDefinition.Image != image.Reference {
|
||||
return errors.New("backup image configuration does not match the installation")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func restoreFilePayload(_ context.Context, installation config.Installation, entry ArchiveEntryMetadata, input io.Reader) error {
|
||||
if entry.Size < 0 || uint64(entry.Size) > defaultPreflightMaxUncompressedBytes {
|
||||
return errors.New("restore file size is invalid")
|
||||
}
|
||||
contents, err := io.ReadAll(io.LimitReader(input, entry.Size+1))
|
||||
if err != nil || int64(len(contents)) != entry.Size {
|
||||
return errors.New("restore file payload is invalid")
|
||||
}
|
||||
target, err := restoreFileTarget(installation, entry)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := replaceRestoreFile(target, contents, os.FileMode(entry.Mode)); err != nil {
|
||||
return errors.New("restore file could not be replaced safely")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func restoreFileTarget(installation config.Installation, entry ArchiveEntryMetadata) (string, error) {
|
||||
if entry.Kind == EntryExternalSecret {
|
||||
return restoreExternalTarget(installation, entry)
|
||||
}
|
||||
if entry.Kind != EntryFile {
|
||||
return "", errors.New("restore file kind is unsupported")
|
||||
}
|
||||
switch entry.Path {
|
||||
case "configuration/installation/thothii-installation.yaml":
|
||||
return installation.Path, nil
|
||||
case "configuration/environment/operator.env":
|
||||
return installation.EnvFile, nil
|
||||
case "configuration/pi/models.json":
|
||||
return filepath.Join(installation.ProjectDirectory, "deploy", "pi", "models.json"), nil
|
||||
case "configuration/pi/settings.json":
|
||||
return filepath.Join(installation.ProjectDirectory, "deploy", "pi", "settings.json"), nil
|
||||
case "configuration/generated/current-image.yaml":
|
||||
return installation.CurrentImageOverridePath(), nil
|
||||
}
|
||||
if strings.HasPrefix(entry.Path, "configuration/overrides/") {
|
||||
name := strings.TrimPrefix(entry.Path, "configuration/overrides/")
|
||||
indexText, base, found := strings.Cut(name, "-")
|
||||
index, parseErr := strconv.Atoi(indexText)
|
||||
if !found || parseErr != nil || len(indexText) != 2 || index < 0 || index >= len(installation.Overrides) || filepath.Base(installation.Overrides[index]) != base {
|
||||
return "", errors.New("restore override target is invalid")
|
||||
}
|
||||
return installation.Overrides[index], nil
|
||||
}
|
||||
if strings.HasPrefix(entry.Owner, "preservation-root:") && strings.HasPrefix(entry.Path, "preservation/") {
|
||||
variable := strings.TrimPrefix(entry.Owner, "preservation-root:")
|
||||
allowed := variable == "THT_DATA_ROOT" || variable == "THT_PI_STATE_ROOT" || variable == "THT_WORKSPACE_REGISTRY_ROOT"
|
||||
parts := strings.SplitN(entry.Path, "/", 3)
|
||||
root, rootErr := installation.EnvironmentValue(variable)
|
||||
if !allowed || len(parts) != 3 || rootErr != nil || root == "" {
|
||||
return "", errors.New("restore preservation target is invalid")
|
||||
}
|
||||
target := filepath.Join(root, filepath.FromSlash(parts[2]))
|
||||
relative, relErr := filepath.Rel(root, target)
|
||||
if relErr != nil || relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) {
|
||||
return "", errors.New("restore preservation target escapes its root")
|
||||
}
|
||||
return target, nil
|
||||
}
|
||||
return "", errors.New("restore file target is not declared")
|
||||
}
|
||||
|
||||
func restoreExternalTarget(installation config.Installation, entry ArchiveEntryMetadata) (string, error) {
|
||||
if entry.SourcePath == "" || filepath.Clean(entry.SourcePath) != entry.SourcePath || !filepath.IsAbs(entry.SourcePath) {
|
||||
return "", errors.New("restore external target is invalid")
|
||||
}
|
||||
if entry.Owner == "external-secret" {
|
||||
paths, err := installation.SecretFiles()
|
||||
if err != nil {
|
||||
return "", errors.New("restore external secret declarations are unavailable")
|
||||
}
|
||||
for _, path := range paths {
|
||||
if path == entry.SourcePath {
|
||||
return path, nil
|
||||
}
|
||||
}
|
||||
return "", errors.New("restore external secret target is not declared")
|
||||
}
|
||||
if entry.Owner == "authentication-configuration" {
|
||||
for _, name := range []string{"auth.yaml", "users.yaml"} {
|
||||
path := filepath.Join(installation.AuthenticationDirectory(), name)
|
||||
if entry.SourcePath == path {
|
||||
return path, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
return "", errors.New("restore external target owner is invalid")
|
||||
}
|
||||
|
||||
func safeRestoreParent(target string) bool {
|
||||
if target == "" || !filepath.IsAbs(target) || filepath.Clean(target) != target {
|
||||
return false
|
||||
}
|
||||
parent := filepath.Dir(target)
|
||||
resolved, err := filepath.EvalSymlinks(parent)
|
||||
if err != nil || resolved != parent {
|
||||
return false
|
||||
}
|
||||
info, err := os.Stat(parent)
|
||||
if err != nil || !info.IsDir() {
|
||||
return false
|
||||
}
|
||||
if targetInfo, err := os.Lstat(target); err == nil {
|
||||
return targetInfo.Mode().IsRegular() && targetInfo.Mode()&os.ModeSymlink == 0
|
||||
} else {
|
||||
return errors.Is(err, os.ErrNotExist)
|
||||
}
|
||||
}
|
||||
|
||||
func volumeRestoreCommand(volume string) []string {
|
||||
return []string{
|
||||
"run", "--rm", "--network", "none", "--mount", "type=volume,src=" + volume + ",dst=/target",
|
||||
helperImage, "sh", "-ceu",
|
||||
"rm -rf -- /target/* /target/.[!.]* /target/..?*; tar --numeric-owner -C /target -xf -",
|
||||
}
|
||||
}
|
||||
|
||||
func restoreVerificationRunning(ctx context.Context, installation config.Installation, runner archiveRunner) (bool, error) {
|
||||
return installationRunning(ctx, installation, runner)
|
||||
}
|
||||
|
||||
func verifyRestoreHealth(ctx context.Context, installation config.Installation, runner archiveRunner) error {
|
||||
running, err := restoreVerificationRunning(ctx, installation, runner)
|
||||
if err != nil || !running {
|
||||
return err
|
||||
}
|
||||
return service.WaitForHealthy(ctx, installation, runner)
|
||||
}
|
||||
|
||||
func verifyRestoreDoctor(ctx context.Context, installation config.Installation, runner archiveRunner) error {
|
||||
running, err := restoreVerificationRunning(ctx, installation, runner)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !running {
|
||||
result, configErr := runner.Run(ctx, installation.ComposeArgs("config", "--quiet"), nil)
|
||||
if configErr != nil {
|
||||
return dockerError("verify restored Compose configuration", result, configErr)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
report, err := doctor.Run(ctx, installation, runner)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !report.OK {
|
||||
return errors.New("aggregate doctor did not pass after restore")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func verifyRestorePi(ctx context.Context, installation config.Installation, runner archiveRunner) error {
|
||||
running, err := restoreVerificationRunning(ctx, installation, runner)
|
||||
if err != nil || !running {
|
||||
return err
|
||||
}
|
||||
return pi.Doctor(ctx, compose.InstallationRunner{Installation: installation, Runner: runner})
|
||||
}
|
||||
|
||||
func verifyRestoreWorkspace(ctx context.Context, installation config.Installation, runner archiveRunner) error {
|
||||
running, err := restoreVerificationRunning(ctx, installation, runner)
|
||||
if err != nil || !running {
|
||||
return err
|
||||
}
|
||||
result, err := runner.Run(ctx, installation.ComposeArgs(
|
||||
"exec", "-T", "core", "curl", "-fsS", "--max-time", "5", "http://127.0.0.1:8787/workspaces",
|
||||
), nil)
|
||||
if err != nil {
|
||||
return dockerError("inspect restored workspaces", result, err)
|
||||
}
|
||||
var workspaces []json.RawMessage
|
||||
if json.Unmarshal([]byte(result.Stdout), &workspaces) != nil {
|
||||
return errors.New("restored workspace inspection returned invalid JSON")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1,6 +1,8 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
@@ -13,6 +15,85 @@ import (
|
||||
"github.com/aritmolab/thothii/tools/tht/internal/config"
|
||||
)
|
||||
|
||||
func TestRestorePublicPathUsesConcreteProductionPreflight(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
installation := config.Installation{
|
||||
Path: filepath.Join(root, "deploy", "local-dev", "thothii-installation.yaml"),
|
||||
ProjectDirectory: root,
|
||||
}
|
||||
missing := filepath.Join(root, "missing.zip")
|
||||
|
||||
_, err := Restore(context.Background(), installation, RestoreRequest{Archive: missing, Confirm: true})
|
||||
|
||||
if err == nil || strings.Contains(err.Error(), "dependencies are unavailable") || !strings.Contains(err.Error(), "backup archive") {
|
||||
t.Fatalf("Restore() error = %v, want production archive preflight", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRestoreRestoresVerifiedVolumesInManifestOrderBeforeAuthenticationReset(t *testing.T) {
|
||||
installation := preflightTestInstallation(t)
|
||||
archive := filepath.Join(t.TempDir(), "restore-volumes.zip")
|
||||
sessionsTar := safeRestoreTar(t, "session.txt", "session")
|
||||
settingsTar := safeRestoreTar(t, "settings.json", "settings")
|
||||
writePreflightArchive(t, archive, preflightArchiveSpec{
|
||||
volumes: []VolumeMetadata{
|
||||
{LogicalName: "sessions", Name: "project_sessions", Driver: "local"},
|
||||
{LogicalName: "settings", Name: "project_settings", Driver: "local"},
|
||||
},
|
||||
entries: []preflightArchiveEntry{
|
||||
{path: "configuration/operator.env", body: []byte("safe")},
|
||||
{path: "volumes/settings.tar", body: settingsTar, kind: EntryVolume, owner: "volume:settings", logicalName: "settings"},
|
||||
{path: "volumes/sessions.tar", body: sessionsTar, kind: EntryVolume, owner: "volume:sessions", logicalName: "sessions"},
|
||||
},
|
||||
})
|
||||
runner := newBackupRunner(installation, false)
|
||||
deps := restoreTestDependencies(t, runner)
|
||||
var events []string
|
||||
deps.restoreFile = func(_ context.Context, _ config.Installation, entry ArchiveEntryMetadata, _ io.Reader) error {
|
||||
events = append(events, "file:"+entry.Path)
|
||||
return nil
|
||||
}
|
||||
deps.restoreVolume = func(_ context.Context, _ config.Installation, volume VolumeMetadata, stream io.Reader) error {
|
||||
if _, err := io.ReadAll(stream); err != nil {
|
||||
return err
|
||||
}
|
||||
events = append(events, "volume:"+volume.LogicalName)
|
||||
return nil
|
||||
}
|
||||
deps.resetAuthenticationState = func(context.Context, config.Installation, archiveRunner) error {
|
||||
events = append(events, "reset-auth-state")
|
||||
return nil
|
||||
}
|
||||
|
||||
if _, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: archive, Confirm: true}, deps); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, want := events, []string{
|
||||
"file:configuration/operator.env",
|
||||
"volume:sessions",
|
||||
"volume:settings",
|
||||
"reset-auth-state",
|
||||
}; !equalStrings(got, want) {
|
||||
t.Fatalf("restore events = %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func safeRestoreTar(t *testing.T, name, contents string) []byte {
|
||||
t.Helper()
|
||||
var output bytes.Buffer
|
||||
writer := tar.NewWriter(&output)
|
||||
if err := writer.WriteHeader(&tar.Header{Name: name, Mode: 0o600, Size: int64(len(contents)), Typeflag: tar.TypeReg}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := writer.Write([]byte(contents)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := writer.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return output.Bytes()
|
||||
}
|
||||
|
||||
func TestRestoreStoppedInstallationRunsCheckpointRestoreAndVerification(t *testing.T) {
|
||||
installation := preflightTestInstallation(t)
|
||||
archive := filepath.Join(t.TempDir(), "restore.zip")
|
||||
@@ -103,7 +184,9 @@ func TestResetAuthenticationStateCreatesOnlyPrivateEmptyStateDirectories(t *test
|
||||
joined := strings.Join(runner.args, "\x00")
|
||||
for _, required := range []string{
|
||||
"run", "--rm", "--no-deps", "--no-TTY", "--entrypoint", "sh", "core", "-ceu",
|
||||
"rm -rf /data/auth && install -d -m 0700 /data/auth /data/auth/sessions /data/auth/oidc",
|
||||
"find /data/auth -mindepth 1 -maxdepth 1 -exec rm -rf -- {} +",
|
||||
"install -d -m 0700 /data/auth /data/auth/sessions /data/auth/oidc",
|
||||
"find /data/auth/sessions /data/auth/oidc -mindepth 1 -print -quit",
|
||||
} {
|
||||
if !strings.Contains(joined, required) {
|
||||
t.Fatalf("authentication state reset command omits %q: %#v", required, runner.args)
|
||||
|
||||
Reference in New Issue
Block a user