mptyl/ThothII
1
0
Fork 0
Code Issues 25 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

feat(cli): add transactional installation backups

Browse Source
This commit is contained in:
marcopan
2026-08-16 00:55:22 +02:00
parent 3e0c864c85
commit 11fbf0a138
8 changed files with 2088 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/tht/cmd/tht/main.go
+105
View File
@@ -14,9 +14,11 @@ import (
"strconv"
"strings"
"github.com/aritmolab/thothii/tools/tht/internal/backup"
"github.com/aritmolab/thothii/tools/tht/internal/compose"
"github.com/aritmolab/thothii/tools/tht/internal/config"
"github.com/aritmolab/thothii/tools/tht/internal/doctor"
"github.com/aritmolab/thothii/tools/tht/internal/lifecycle"
"github.com/aritmolab/thothii/tools/tht/internal/output"
"github.com/aritmolab/thothii/tools/tht/internal/pi"
"github.com/aritmolab/thothii/tools/tht/internal/project"
@@ -42,6 +44,8 @@ Commands:
start [--build] Start the installation; --build builds current-checkout images first.
stop Stop the installation.
update --check-only Validate the current installation without changing containers.
backup [--output PATH] [--include-secrets --yes] [--drain]
Create one transactional installation backup.
sessions migrate --yes
Run only the server session migrator and verify pending=[] and drifted=[].
remove Display exact stopped app container IDs without mutation.
@@ -169,6 +173,8 @@ func run(ctx context.Context, args []string, stdout, stderr io.Writer) int {
return commandUsageError(stderr, "update currently requires --check-only")
}
result, err = runner.Run(ctx, installation.ComposeArgs("config", "--quiet"), nil)
case "backup":
return backupCommand(ctx, installation, commandArgs, stdout, stderr)
case "doctor":
return doctorCommand(ctx, installation, runner, commandArgs, stdout, stderr)
case "pi":
@@ -419,6 +425,13 @@ func piCommand(ctx context.Context, installation config.Installation, runner com
if len(args) == 0 {
return commandUsageError(stderr, "pi requires a subcommand")
}
if piMutationRequiresLifecycleLock(args) {
lock, err := lifecycle.Acquire(installation)
if err != nil {
return lifecycleFailure(stderr, err, secretValues)
}
defer func() { _ = lock.Release() }()
}
controlled := compose.InstallationRunner{Installation: installation, Runner: runner}
switch args[0] {
case "status":
@@ -790,6 +803,98 @@ func logsArgs(args []string) ([]string, error) {
return nil, errors.New("logs does not accept arguments; use bounded snapshots")
}
type backupExecutor func(context.Context, config.Installation, backup.CreateRequest) (backup.Result, error)
func backupCommand(ctx context.Context, installation config.Installation, args []string, stdout, stderr io.Writer) int {
return backupCommandWith(ctx, installation, args, backup.Create, stdout, stderr)
}
func backupCommandWith(
ctx context.Context,
installation config.Installation,
args []string,
execute backupExecutor,
stdout, stderr io.Writer,
) int {
request, err := parseBackupArgs(args)
if err != nil {
return commandUsageError(stderr, err.Error())
}
if execute == nil {
return commandUsageError(stderr, "backup executor is unavailable")
}
result, err := execute(ctx, installation, request)
if err != nil {
message := output.Sanitize(err.Error(), nil)
fmt.Fprintf(stderr, "tht: %s\n", message)
if errors.Is(err, backup.ErrActiveSessions) || errors.Is(err, backup.ErrConfirmationRequired) || errors.Is(err, lifecycle.ErrLocked) {
return 2
}
return 1
}
fmt.Fprintf(stdout, "Backup created: %s\n", result.Path)
if result.Warning != "" {
fmt.Fprintln(stderr, output.Sanitize(result.Warning, nil))
}
return 0
}
func parseBackupArgs(args []string) (backup.CreateRequest, error) {
request := backup.CreateRequest{}
for len(args) > 0 {
option := args[0]
args = args[1:]
switch option {
case "--output":
if request.Output != "" {
return backup.CreateRequest{}, errors.New("--output may be supplied once")
}
if len(args) == 0 || args[0] == "" {
return backup.CreateRequest{}, errors.New("--output requires a path")
}
request.Output, args = args[0], args[1:]
case "--include-secrets":
if request.IncludeSecrets {
return backup.CreateRequest{}, errors.New("--include-secrets may be supplied once")
}
request.IncludeSecrets = true
case "--yes":
if request.Confirm {
return backup.CreateRequest{}, errors.New("--yes may be supplied once")
}
request.Confirm = true
case "--drain":
if request.Drain {
return backup.CreateRequest{}, errors.New("--drain may be supplied once")
}
request.Drain = true
default:
return backup.CreateRequest{}, fmt.Errorf("unknown backup option %q", option)
}
}
if request.IncludeSecrets && !request.Confirm {
return backup.CreateRequest{}, errors.New("--include-secrets requires --yes")
}
if request.Confirm && !request.IncludeSecrets {
return backup.CreateRequest{}, errors.New("--yes is only valid with --include-secrets")
}
return request, nil
}
func piMutationRequiresLifecycleLock(args []string) bool {
if len(args) == 0 {
return false
}
switch args[0] {
case "configure", "restart", "update", "rollback":
return true
case "maintenance":
return len(args) > 1 && args[1] == "recover"
default:
return false
}
}
func commandUsageError(stderr io.Writer, message string) int {
fmt.Fprintf(stderr, "tht: %s\n", message)
return 2
tools/tht/cmd/tht/main_test.go
+75
View File
@@ -12,6 +12,7 @@ import (
"strings"
"testing"
"github.com/aritmolab/thothii/tools/tht/internal/backup"
"github.com/aritmolab/thothii/tools/tht/internal/compose"
"github.com/aritmolab/thothii/tools/tht/internal/config"
"github.com/aritmolab/thothii/tools/tht/internal/doctor"
@@ -20,6 +21,80 @@ import (
"github.com/aritmolab/thothii/tools/tht/internal/testsupport"
)
func TestBackupCommandParsesSafeTransactionalOptions(t *testing.T) {
request, err := parseBackupArgs([]string{"--output", "backup.zip", "--include-secrets", "--yes", "--drain"})
if err != nil {
t.Fatal(err)
}
if request.Output != "backup.zip" || !request.IncludeSecrets || !request.Confirm || !request.Drain {
t.Fatalf("backup request = %#v", request)
}
for _, args := range [][]string{
{"--include-secrets"},
{"--yes"},
{"--output"},
{"--drain", "--drain"},
{"--unknown"},
} {
if _, err := parseBackupArgs(args); err == nil {
t.Errorf("parseBackupArgs(%v) succeeded", args)
}
}
}
func TestBackupCommandDispatchesWithoutDockerAndPrintsCustodyWarning(t *testing.T) {
installation := config.Installation{Path: "/tmp/thothii-installation.yaml"}
var received backup.CreateRequest
executor := func(_ context.Context, got config.Installation, request backup.CreateRequest) (backup.Result, error) {
if got.Path != installation.Path {
t.Fatalf("installation = %#v", got)
}
received = request
return backup.Result{Path: "/tmp/backup.zip", Warning: "archive contains external secret files; protect its custody"}, nil
}
var stdout, stderr bytes.Buffer
code := backupCommandWith(context.Background(), installation, []string{"--include-secrets", "--yes"}, executor, &stdout, &stderr)
if code != 0 {
t.Fatalf("backup exit = %d, stderr = %s", code, stderr.String())
}
if !received.IncludeSecrets || !received.Confirm || !strings.Contains(stdout.String(), "/tmp/backup.zip") || !strings.Contains(stderr.String(), "protect its custody") {
t.Fatalf("request=%#v stdout=%q stderr=%q", received, stdout.String(), stderr.String())
}
}
func TestBackupCommandMapsSafetyRefusalsToUsageExit(t *testing.T) {
executor := func(context.Context, config.Installation, backup.CreateRequest) (backup.Result, error) {
return backup.Result{}, backup.ErrActiveSessions
}
var stdout, stderr bytes.Buffer
if code := backupCommandWith(context.Background(), config.Installation{}, nil, executor, &stdout, &stderr); code != 2 {
t.Fatalf("backup safety refusal exit = %d, stderr = %q", code, stderr.String())
}
}
func TestPiMutationsUseTheSharedInstallationLifecycleLock(t *testing.T) {
for _, test := range []struct {
args []string
want bool
}{
{args: []string{"status"}, want: false},
{args: []string{"doctor"}, want: false},
{args: []string{"test"}, want: false},
{args: []string{"logs"}, want: false},
{args: []string{"configure"}, want: true},
{args: []string{"restart"}, want: true},
{args: []string{"update"}, want: true},
{args: []string{"rollback"}, want: true},
{args: []string{"maintenance", "recover"}, want: true},
{args: []string{"maintenance", "status"}, want: false},
} {
if got := piMutationRequiresLifecycleLock(test.args); got != test.want {
t.Errorf("piMutationRequiresLifecycleLock(%v) = %v, want %v", test.args, got, test.want)
}
}
}
func TestInstallationRunnerMapsProfileToSessionInventoryScope(t *testing.T) {
for _, test := range []struct {
profile string
tools/tht/internal/backup/create.go
+761
View File
@@ -0,0 +1,761 @@
// Package backup creates portable, transactional ThothII installation archives.
package backup
import (
"archive/zip"
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
"sort"
"strings"
"time"
"github.com/aritmolab/thothii/tools/tht/internal/compose"
"github.com/aritmolab/thothii/tools/tht/internal/config"
"github.com/aritmolab/thothii/tools/tht/internal/lifecycle"
"github.com/aritmolab/thothii/tools/tht/internal/service"
)
var (
// ErrActiveSessions keeps the refusal machine-readable for the host CLI.
ErrActiveSessions = errors.New("active sessions require --drain before backup")
// ErrConfirmationRequired guards backups that would contain external secret payloads.
ErrConfirmationRequired = errors.New("--include-secrets requires --yes")
)
var requiredVolumes = []string{
"settings",
"pi-state",
"workspace-registry",
"workspace-secrets",
"sessions",
"qdrant-data",
"embedding-models",
}
const (
helperImage = "busybox:1.36.1"
drainPollInterval = time.Second
maxDrainPolls = 300
)
// CreateRequest controls one explicit backup request.
type CreateRequest struct {
Output string
IncludeSecrets bool
Confirm bool
Drain bool
}
// Result describes a published archive without exposing its contents.
type Result struct {
Path string
Warning string
}
// archiveRunner separates the only streaming Docker call from regular Compose commands. Tests
// provide a deterministic in-memory implementation; production always passes argument arrays.
type archiveRunner interface {
compose.Runner
Stream(context.Context, []string, io.Reader, io.Writer) (compose.Result, error)
SessionInventoryScope() string
}
type dependencies struct {
runner archiveRunner
now func() time.Time
homeDir func() (string, error)
revision func(context.Context, string) (string, error)
sleep func(time.Duration)
rename func(string, string) error
}
// Create creates an archive with the real Docker command boundary. It performs no shell
// interpolation and never supplies secret contents in process arguments.
func Create(ctx context.Context, installation config.Installation, request CreateRequest) (Result, error) {
runner := hostRunner{runner: compose.NewRunner(""), binary: "docker", profile: installation.Profile}
return createWithDependencies(ctx, installation, request, dependencies{
runner: runner,
now: time.Now,
homeDir: os.UserHomeDir,
revision: func(ctx context.Context, directory string) (string, error) {
command := exec.CommandContext(ctx, "git", "-C", directory, "rev-parse", "HEAD")
value, err := command.Output()
if err != nil {
return "", errors.New("source revision is unavailable")
}
return strings.TrimSpace(string(value)), nil
},
sleep: time.Sleep,
rename: os.Rename,
})
}
func createWithDependencies(ctx context.Context, installation config.Installation, request CreateRequest, dependencies dependencies) (result Result, resultErr error) {
if dependencies.runner == nil || dependencies.now == nil || dependencies.homeDir == nil || dependencies.revision == nil || dependencies.sleep == nil || dependencies.rename == nil {
return Result{}, errors.New("backup dependencies are incomplete")
}
if request.IncludeSecrets && !request.Confirm {
return Result{}, ErrConfirmationRequired
}
lock, err := lifecycle.Acquire(installation)
if err != nil {
return Result{}, err
}
defer func() {
if releaseErr := lock.Release(); releaseErr != nil && resultErr == nil {
result = Result{}
resultErr = releaseErr
}
}()
if err := rejectInlineSecretValues(installation.EnvFile); err != nil {
return Result{}, err
}
installationID, err := backupInstallationID(installation)
if err != nil {
return Result{}, err
}
secretPaths, err := installation.SecretFiles()
if err != nil {
return Result{}, fmt.Errorf("installation external secret references could not be read: %w", err)
}
revision, err := dependencies.revision(ctx, installation.ProjectDirectory)
if err != nil {
return Result{}, err
}
if !revisionPattern.MatchString(revision) {
return Result{}, errors.New("source revision is invalid")
}
output, err := backupOutputPath(request.Output, installationID, revision, dependencies.now().UTC(), dependencies.homeDir)
if err != nil {
return Result{}, err
}
if err := ensureNewArchivePath(output); err != nil {
return Result{}, err
}
rendered, err := renderedConfiguration(ctx, installation, dependencies.runner)
if err != nil {
return Result{}, err
}
volumes, err := inspectRequiredVolumes(ctx, dependencies.runner, rendered)
if err != nil {
return Result{}, err
}
images, err := imageIdentities(ctx, installation, dependencies.runner, rendered)
if err != nil {
return Result{}, err
}
wasRunning, err := installationRunning(ctx, installation, dependencies.runner)
if err != nil {
return Result{}, err
}
maintenanceActive := false
stopped := false
defer func() {
if stopped {
if startErr := composeStartAndVerify(ctx, installation, dependencies.runner); startErr != nil && resultErr == nil {
result = Result{}
resultErr = startErr
}
}
if maintenanceActive {
if deactivateErr := maintenance(ctx, installation, dependencies.runner, false); deactivateErr != nil && resultErr == nil {
result = Result{}
resultErr = deactivateErr
}
}
}()
if wasRunning {
if err := maintenance(ctx, installation, dependencies.runner, true); err != nil {
return Result{}, err
}
maintenanceActive = true
if err := waitForNoActiveSessions(ctx, installation, dependencies.runner, request.Drain, dependencies.sleep); err != nil {
return Result{}, err
}
if err := runCompose(ctx, installation, dependencies.runner, "stop"); err != nil {
return Result{}, err
}
stopped = true
}
manifest := Manifest{
SchemaVersion: CurrentSchemaVersion,
InstallationID: installationID,
CreatedAt: dependencies.now().UTC(),
SourceRevision: revision,
IncludesSecrets: request.IncludeSecrets && len(secretPaths) > 0,
ComposeProject: installation.ProjectName(),
Images: images,
Volumes: volumes,
}
if err := writeArchive(ctx, output, installation, request, secretPaths, manifest, volumes, dependencies); err != nil {
return Result{}, err
}
if wasRunning {
if err := composeStartAndVerify(ctx, installation, dependencies.runner); err != nil {
return Result{}, err
}
stopped = false
if err := maintenance(ctx, installation, dependencies.runner, false); err != nil {
return Result{}, err
}
maintenanceActive = false
}
result = Result{Path: output}
if request.IncludeSecrets {
result.Warning = "The archive contains external secret files. Protect its custody and access."
}
return result, nil
}
type hostRunner struct {
runner compose.Runner
binary string
profile string
}
func (runner hostRunner) Run(ctx context.Context, args []string, stdin io.Reader) (compose.Result, error) {
return runner.runner.Run(ctx, args, stdin)
}
func (runner hostRunner) Stream(ctx context.Context, args []string, stdin io.Reader, stdout io.Writer) (compose.Result, error) {
command := exec.CommandContext(ctx, runner.binary, args...)
command.Stdin = stdin
command.Stdout = stdout
var stderr bytes.Buffer
command.Stderr = &stderr
err := command.Run()
result := compose.Result{Stderr: stderr.String()}
if exitErr := new(exec.ExitError); errors.As(err, &exitErr) {
result.ExitCode = exitErr.ExitCode()
}
return result, err
}
func (runner hostRunner) SessionInventoryScope() string {
if runner.profile == "local" {
return "mine"
}
return "all"
}
func backupInstallationID(installation config.Installation) (string, error) {
value := filepath.Base(filepath.Dir(installation.Path))
if value == "" || value == "." || value == ".." || strings.ContainsAny(value, `/\\`) {
return "", errors.New("installation ID is invalid")
}
return value, nil
}
func backupOutputPath(requested, installationID, revision string, now time.Time, homeDir func() (string, error)) (string, error) {
if requested != "" {
if !filepath.IsAbs(requested) {
absolute, err := filepath.Abs(requested)
if err != nil {
return "", errors.New("backup output path is unavailable")
}
requested = absolute
}
return filepath.Clean(requested), nil
}
home, err := homeDir()
if err != nil || home == "" {
return "", errors.New("home directory is unavailable for the default backup path")
}
name := fmt.Sprintf("thothii-%s-%s-%s.zip", installationID, now.UTC().Format("20060102T150405Z"), revision)
return filepath.Join(home, ".thothii", "backups", installationID, name), nil
}
func ensureNewArchivePath(output string) error {
if output == "" || !filepath.IsAbs(output) {
return errors.New("backup output path must be absolute")
}
if err := os.MkdirAll(filepath.Dir(output), 0o700); err != nil {
return fmt.Errorf("create backup directory: %w", err)
}
if info, err := os.Lstat(output); err == nil || !errors.Is(err, os.ErrNotExist) {
if err == nil && info.Mode().IsRegular() {
return errors.New("backup output already exists")
}
return errors.New("backup output path is unavailable")
}
return nil
}
type renderedCompose struct {
Volumes map[string]struct {
Name string `json:"name"`
} `json:"volumes"`
Services map[string]struct {
Image string `json:"image"`
} `json:"services"`
}
func renderedConfiguration(ctx context.Context, installation config.Installation, runner archiveRunner) (renderedCompose, error) {
result, err := runner.Run(ctx, installation.ComposeArgs("config", "--format", "json"), nil)
if err != nil {
return renderedCompose{}, dockerError("render Compose configuration", result, err)
}
var rendered renderedCompose
if json.Unmarshal([]byte(result.Stdout), &rendered) != nil {
return renderedCompose{}, errors.New("Docker Compose returned invalid backup configuration")
}
for _, logical := range requiredVolumes {
if rendered.Volumes[logical].Name == "" {
return renderedCompose{}, fmt.Errorf("required backup volume %q is not configured", logical)
}
}
return rendered, nil
}
func inspectRequiredVolumes(ctx context.Context, runner archiveRunner, rendered renderedCompose) ([]VolumeMetadata, error) {
names := make([]string, 0, len(requiredVolumes))
for _, logical := range requiredVolumes {
names = append(names, rendered.Volumes[logical].Name)
}
result, err := runner.Run(ctx, append([]string{"volume", "inspect"}, names...), nil)
if err != nil {
return nil, dockerError("inspect backup volumes", result, err)
}
var inspected []struct {
Name string `json:"Name"`
Driver string `json:"Driver"`
Labels map[string]string `json:"Labels"`
}
if json.Unmarshal([]byte(result.Stdout), &inspected) != nil {
return nil, errors.New("Docker returned invalid volume metadata")
}
byName := make(map[string]struct {
Name string
Driver string
Labels map[string]string
}, len(inspected))
for _, item := range inspected {
byName[item.Name] = struct {
Name string
Driver string
Labels map[string]string
}{item.Name, item.Driver, item.Labels}
}
volumes := make([]VolumeMetadata, 0, len(requiredVolumes))
for _, logical := range requiredVolumes {
name := rendered.Volumes[logical].Name
item, ok := byName[name]
if !ok || item.Driver == "" {
return nil, fmt.Errorf("required backup volume %q is unavailable", logical)
}
volumes = append(volumes, VolumeMetadata{LogicalName: logical, Name: name, Driver: item.Driver, Labels: item.Labels})
}
return volumes, nil
}
func imageIdentities(ctx context.Context, installation config.Installation, runner archiveRunner, rendered renderedCompose) ([]ImageIdentity, error) {
result, err := runner.Run(ctx, installation.ComposeArgs("images", "--format", "json"), nil)
if err != nil {
return nil, dockerError("inspect image identities", result, err)
}
ids := map[string]string{}
decoder := json.NewDecoder(strings.NewReader(result.Stdout))
for {
var item struct {
Service string `json:"Service"`
ID string `json:"ID"`
}
err := decoder.Decode(&item)
if errors.Is(err, io.EOF) {
break
}
if err != nil || item.Service == "" {
return nil, errors.New("Docker Compose returned invalid image identities")
}
ids[item.Service] = item.ID
}
services := make([]string, 0, len(rendered.Services))
for name, definition := range rendered.Services {
if definition.Image != "" {
services = append(services, name)
}
}
sort.Strings(services)
images := make([]ImageIdentity, 0, len(services))
for _, name := range services {
images = append(images, ImageIdentity{Service: name, Reference: rendered.Services[name].Image, ID: ids[name]})
}
return images, nil
}
func installationRunning(ctx context.Context, installation config.Installation, runner archiveRunner) (bool, error) {
result, err := runner.Run(ctx, installation.ComposeArgs("ps", "--status", "running", "-q"), nil)
if err != nil {
return false, dockerError("inspect running services", result, err)
}
return strings.TrimSpace(result.Stdout) != "", nil
}
func maintenance(ctx context.Context, installation config.Installation, runner archiveRunner, activate bool) error {
action := "deactivate"
if activate {
action = "activate"
}
result, err := runner.Run(ctx, installation.ComposeArgs(
"exec", "-T", "core", "curl", "-fsS", "--max-time", "5", "-X", "POST",
"http://127.0.0.1:8787/internal/maintenance/"+action,
), nil)
if err != nil {
return dockerError("change maintenance admissions", result, err)
}
return nil
}
func waitForNoActiveSessions(ctx context.Context, installation config.Installation, runner archiveRunner, drain bool, sleep func(time.Duration)) error {
for attempt := 0; attempt < maxDrainPolls; attempt++ {
result, err := runner.Run(ctx, installation.ComposeArgs(
"exec", "-T", "core", "curl", "-fsS", "--max-time", "5",
"http://127.0.0.1:8787/sessions?scope="+runner.SessionInventoryScope(),
), nil)
if err != nil {
return dockerError("inspect active sessions", result, err)
}
active, err := activeSessions(result.Stdout)
if err != nil {
return err
}
if !active {
return nil
}
if !drain {
return ErrActiveSessions
}
select {
case <-ctx.Done():
return ctx.Err()
default:
sleep(drainPollInterval)
}
}
return errors.New("active sessions did not drain before the backup deadline")
}
func activeSessions(value string) (bool, error) {
var sessions []struct {
Status string `json:"status"`
Archived bool `json:"archived"`
}
if json.Unmarshal([]byte(value), &sessions) != nil {
return false, errors.New("session inventory is invalid")
}
for _, session := range sessions {
if !session.Archived && (session.Status == "" || session.Status == "running" || session.Status == "active") {
return true, nil
}
}
return false, nil
}
func composeStartAndVerify(ctx context.Context, installation config.Installation, runner archiveRunner) error {
if err := runCompose(ctx, installation, runner, "start"); err != nil {
return err
}
return service.WaitForHealthy(ctx, installation, runner)
}
func runCompose(ctx context.Context, installation config.Installation, runner archiveRunner, command ...string) error {
result, err := runner.Run(ctx, installation.ComposeArgs(command...), nil)
if err != nil {
return dockerError("run Docker Compose", result, err)
}
return nil
}
func writeArchive(ctx context.Context, output string, installation config.Installation, request CreateRequest, secretPaths []string, manifest Manifest, volumes []VolumeMetadata, dependencies dependencies) (resultErr error) {
directory := filepath.Dir(output)
temporary, err := os.CreateTemp(directory, ".tht-backup-*.tmp")
if err != nil {
return fmt.Errorf("create temporary backup archive: %w", err)
}
temporaryPath := temporary.Name()
published := false
closedFile := false
defer func() {
var closeErr error
if !closedFile {
closeErr = temporary.Close()
}
if closeErr != nil && resultErr == nil {
resultErr = closeErr
}
if !published {
_ = os.Remove(temporaryPath)
_ = os.Remove(output)
}
}()
if err := temporary.Chmod(0o600); err != nil {
return fmt.Errorf("protect temporary backup archive: %w", err)
}
writer := zip.NewWriter(temporary)
closedWriter := false
defer func() {
if !closedWriter {
_ = writer.Close()
}
}()
add := func(name string, mode os.FileMode, input io.Reader) (Entry, error) {
header := &zip.FileHeader{Name: filepath.ToSlash(name), Method: zip.Deflate}
header.SetMode(mode)
created, err := writer.CreateHeader(header)
if err != nil {
return Entry{}, err
}
hash := sha256.New()
size, err := io.Copy(io.MultiWriter(created, hash), input)
if err != nil {
return Entry{}, err
}
return Entry{Path: filepath.ToSlash(name), SHA256: "sha256:" + hex.EncodeToString(hash.Sum(nil)), Size: size, Archived: true, Mode: uint32(mode.Perm())}, nil
}
addFile := func(name, owner string, sensitive bool, source string) error {
file, err := os.Open(source)
if err != nil {
return fmt.Errorf("read backup input: %w", err)
}
defer file.Close()
info, err := file.Stat()
if err != nil || !info.Mode().IsRegular() {
return errors.New("backup input is not a regular file")
}
entry, err := add(name, info.Mode(), file)
if err != nil {
return err
}
entry.Kind, entry.Owner, entry.Sensitive = EntryFile, owner, sensitive
manifest.Entries = append(manifest.Entries, entry)
return nil
}
for _, input := range configurationInputs(installation) {
if input.optional {
if _, err := os.Stat(input.source); errors.Is(err, os.ErrNotExist) {
continue
}
}
if err := addFile(input.name, input.owner, false, input.source); err != nil {
return err
}
}
for index, source := range secretPaths {
if request.IncludeSecrets {
if err := addFile(fmt.Sprintf("external-secrets/%03d", index), "external-secret", true, source); err != nil {
return err
}
entry := &manifest.Entries[len(manifest.Entries)-1]
entry.Kind, entry.SourcePath, entry.Sensitive = EntryExternalSecret, source, true
continue
}
file, err := os.Open(source)
if err != nil {
return errors.New("external secret file is unavailable")
}
info, statErr := file.Stat()
if statErr != nil || !info.Mode().IsRegular() {
_ = file.Close()
return errors.New("external secret file is unavailable")
}
hash := sha256.New()
size, copyErr := io.Copy(hash, file)
closeErr := file.Close()
if copyErr != nil || closeErr != nil {
return errors.New("external secret file is unavailable")
}
manifest.Entries = append(manifest.Entries, Entry{
Path: fmt.Sprintf("external-secrets/%03d", index), Kind: EntrySecretReference, Owner: "external-secret",
SourcePath: source, SHA256: "sha256:" + hex.EncodeToString(hash.Sum(nil)), Size: size, Sensitive: true,
})
}
for _, volume := range volumes {
headerName := "volumes/" + volume.LogicalName + ".tar"
header := &zip.FileHeader{Name: headerName, Method: zip.Deflate}
header.SetMode(0o600)
archiveWriter, err := writer.CreateHeader(header)
if err != nil {
return fmt.Errorf("create volume archive entry %s: %w", volume.LogicalName, err)
}
hash := sha256.New()
sizeWriter := &countingWriter{writer: io.MultiWriter(archiveWriter, hash)}
streamResult, streamErr := dependencies.runner.Stream(ctx, volumeArchiveCommand(volume.Name), nil, sizeWriter)
if streamErr != nil || streamResult.ExitCode != 0 {
if streamErr == nil {
streamErr = errors.New("Docker volume helper returned a nonzero exit status")
}
return fmt.Errorf("archive volume %s: %w", volume.LogicalName, dockerError("stream volume", streamResult, streamErr))
}
entry := Entry{
Path: headerName, Kind: EntryVolume, Owner: "volume:" + volume.LogicalName, LogicalName: volume.LogicalName,
SHA256: "sha256:" + hex.EncodeToString(hash.Sum(nil)), Size: sizeWriter.size, Mode: 0o600, Archived: true,
Sensitive: volume.LogicalName == "workspace-secrets",
}
manifest.Entries = append(manifest.Entries, entry)
}
if installation.Profile == "server" {
if err := archivePreservationRoots(installation, secretPaths, addFile); err != nil {
return err
}
}
manifestJSON, err := manifest.JSON()
if err != nil {
return err
}
if _, err := add(ManifestPath, 0o600, bytes.NewReader(manifestJSON)); err != nil {
return err
}
if err := writer.Close(); err != nil {
return fmt.Errorf("close backup archive: %w", err)
}
closedWriter = true
if err := temporary.Sync(); err != nil {
return fmt.Errorf("fsync backup archive: %w", err)
}
if err := temporary.Close(); err != nil {
return fmt.Errorf("close backup archive: %w", err)
}
closedFile = true
if err := dependencies.rename(temporaryPath, output); err != nil {
return fmt.Errorf("publish backup archive: %w", err)
}
published = true
return nil
}
type configurationInput struct {
name, owner, source string
optional bool
}
func configurationInputs(installation config.Installation) []configurationInput {
inputs := []configurationInput{
{"configuration/installation/thothii-installation.yaml", "installation", installation.Path, false},
{"configuration/environment/operator.env", "installation", installation.EnvFile, false},
{"configuration/pi/models.json", "pi", filepath.Join(installation.ProjectDirectory, "deploy", "pi", "models.json"), false},
{"configuration/pi/settings.json", "pi", filepath.Join(installation.ProjectDirectory, "deploy", "pi", "settings.json"), false},
{"configuration/generated/current-image.yaml", "installation", installation.CurrentImageOverridePath(), true},
}
for index, source := range installation.Overrides {
inputs = append(inputs, configurationInput{fmt.Sprintf("configuration/overrides/%02d-%s", index, filepath.Base(source)), "installation", source, false})
}
return inputs
}
func volumeArchiveCommand(volume string) []string {
return []string{"run", "--rm", "--network", "none", "--mount", "type=volume,src=" + volume + ",dst=/source,readonly", helperImage, "tar", "--numeric-owner", "-C", "/source", "-cf", "-", "."}
}
type countingWriter struct {
writer io.Writer
size int64
}
func (writer *countingWriter) Write(value []byte) (int, error) {
written, err := writer.writer.Write(value)
writer.size += int64(written)
return written, err
}
func archivePreservationRoots(installation config.Installation, secretPaths []string, addFile func(string, string, bool, string) error) error {
backupRoot, err := installation.EnvironmentValue("THT_BACKUP_ROOT")
if err != nil {
return errors.New("server preservation roots are unavailable")
}
secretSet := make(map[string]bool, len(secretPaths))
for _, secret := range secretPaths {
secretSet[secret] = true
}
for index, variable := range []string{"THT_DATA_ROOT", "THT_PI_STATE_ROOT", "THT_WORKSPACE_REGISTRY_ROOT"} {
root, err := installation.EnvironmentValue(variable)
if err != nil || root == "" {
return errors.New("server preservation roots are unavailable")
}
info, err := os.Stat(root)
if err != nil || !info.IsDir() {
return errors.New("server preservation root is unavailable")
}
err = filepath.WalkDir(root, func(path string, entry os.DirEntry, walkErr error) error {
if walkErr != nil {
return walkErr
}
if path == backupRoot {
if entry.IsDir() {
return filepath.SkipDir
}
return nil
}
if entry.Type()&os.ModeSymlink != 0 {
return errors.New("server preservation root contains a symlink")
}
if entry.IsDir() || secretSet[path] {
return nil
}
if !entry.Type().IsRegular() {
return errors.New("server preservation root contains an unsupported entry")
}
relative, err := filepath.Rel(root, path)
if err != nil {
return err
}
name := fmt.Sprintf("preservation/%02d-%s/%s", index, filepath.Base(root), filepath.ToSlash(relative))
return addFile(name, "preservation-root:"+variable, false, path)
})
if err != nil {
return err
}
}
return nil
}
func rejectInlineSecretValues(environmentPath string) error {
contents, err := os.ReadFile(environmentPath)
if err != nil {
return errors.New("installation environment could not be read")
}
for _, line := range strings.Split(string(contents), "\n") {
line = strings.TrimSpace(line)
if line == "" || strings.HasPrefix(line, "#") {
continue
}
line = strings.TrimPrefix(line, "export ")
key, value, found := strings.Cut(line, "=")
if !found || strings.TrimSpace(value) == "" {
continue
}
key = strings.ToUpper(strings.TrimSpace(key))
if strings.HasSuffix(key, "_FILE") || strings.HasSuffix(key, "_SOURCE") {
continue
}
if strings.Contains(key, "SECRET") || strings.Contains(key, "PASSWORD") || strings.Contains(key, "TOKEN") || strings.Contains(key, "API_KEY") || strings.Contains(key, "CREDENTIAL") || strings.Contains(key, "AUTHORIZATION") || strings.HasSuffix(key, "_AUTH") {
return errors.New("installation environment contains an inline secret value")
}
}
return nil
}
func dockerError(action string, result compose.Result, err error) error {
if result.ExitCode != 0 {
return fmt.Errorf("%s: Docker exited with status %d", action, result.ExitCode)
}
if err != nil {
return fmt.Errorf("%s: %w", action, err)
}
return fmt.Errorf("%s failed", action)
}
tools/tht/internal/backup/create_test.go
+638
View File
@@ -0,0 +1,638 @@
package backup
import (
"archive/tar"
"archive/zip"
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"sort"
"strings"
"testing"
"time"
"github.com/aritmolab/thothii/tools/tht/internal/compose"
"github.com/aritmolab/thothii/tools/tht/internal/config"
"github.com/aritmolab/thothii/tools/tht/internal/lifecycle"
)
var requiredTestVolumes = []string{"settings", "pi-state", "workspace-registry", "workspace-secrets", "sessions", "qdrant-data", "embedding-models"}
func TestCreateWritesManifestLastWithConfigurationMetadataAndSevenVolumes(t *testing.T) {
fixture := newBackupFixture(t, "local")
output := filepath.Join(t.TempDir(), "custom.zip")
runner := newBackupRunner(fixture.installation, false)
result, err := createWithDependencies(context.Background(), fixture.installation, CreateRequest{Output: output}, testDependencies(t, runner))
if err != nil {
t.Fatal(err)
}
if result.Path != output {
t.Fatalf("Result.Path = %q, want %q", result.Path, output)
}
archive := readFixtureArchive(t, output)
if got := archive.order[len(archive.order)-1]; got != ManifestPath {
t.Fatalf("last archive entry = %q, want %q", got, ManifestPath)
}
if archive.manifest.InstallationID != fixture.installationID || archive.manifest.SourceRevision != testRevision || archive.manifest.ComposeProject != fixture.installation.ProjectName() {
t.Fatalf("manifest identity = %#v", archive.manifest)
}
if len(archive.manifest.Volumes) != len(requiredTestVolumes) {
t.Fatalf("manifest volumes = %d, want %d", len(archive.manifest.Volumes), len(requiredTestVolumes))
}
for _, logical := range requiredTestVolumes {
path := "volumes/" + logical + ".tar"
if _, exists := archive.files[path]; !exists {
t.Errorf("archive is missing %s", path)
}
}
for _, path := range []string{
"configuration/installation/thothii-installation.yaml",
"configuration/environment/operator.env",
"configuration/pi/models.json",
"configuration/pi/settings.json",
"configuration/generated/current-image.yaml",
} {
if _, exists := archive.files[path]; !exists {
t.Errorf("archive is missing %s", path)
}
}
if len(archive.manifest.Images) < 2 {
t.Fatalf("image identities = %#v", archive.manifest.Images)
}
if runner.streamWhileRunning {
t.Fatal("a volume was streamed before the installation was stopped")
}
}
func TestCreateRestartsAndVerifiesAnInstallationThatWasRunning(t *testing.T) {
fixture := newBackupFixture(t, "local")
runner := newBackupRunner(fixture.installation, true)
output := filepath.Join(t.TempDir(), "running.zip")
if _, err := createWithDependencies(context.Background(), fixture.installation, CreateRequest{Output: output}, testDependencies(t, runner)); err != nil {
t.Fatal(err)
}
if !runner.running || !runner.coreRunning || runner.maintenance {
t.Fatalf("running state was not restored: running=%v core=%v maintenance=%v", runner.running, runner.coreRunning, runner.maintenance)
}
if runner.stopCount != 1 || runner.startCount != 1 || runner.healthChecks == 0 {
t.Fatalf("lifecycle counts: stop=%d start=%d health=%d", runner.stopCount, runner.startCount, runner.healthChecks)
}
}
func TestCreateRefusesActiveSessionsWithoutDrainAndRestoresAdmissions(t *testing.T) {
fixture := newBackupFixture(t, "local")
runner := newBackupRunner(fixture.installation, true)
runner.sessionResponses = []string{activeSessionPayload()}
output := filepath.Join(t.TempDir(), "refused.zip")
_, err := createWithDependencies(context.Background(), fixture.installation, CreateRequest{Output: output}, testDependencies(t, runner))
if !errors.Is(err, ErrActiveSessions) {
t.Fatalf("Create() error = %v, want ErrActiveSessions", err)
}
if runner.stopCount != 0 || !runner.running || runner.maintenance {
t.Fatalf("refusal changed lifecycle state: stop=%d running=%v maintenance=%v", runner.stopCount, runner.running, runner.maintenance)
}
if _, statErr := os.Stat(output); !errors.Is(statErr, os.ErrNotExist) {
t.Fatalf("unsafe backup was published: %v", statErr)
}
}
func TestCreateDrainsActiveSessionsBeforeStoppedSnapshot(t *testing.T) {
fixture := newBackupFixture(t, "local")
runner := newBackupRunner(fixture.installation, true)
runner.sessionResponses = []string{activeSessionPayload(), activeSessionPayload(), `[]`}
dependencies := testDependencies(t, runner)
sleeps := 0
dependencies.sleep = func(time.Duration) { sleeps++ }
if _, err := createWithDependencies(context.Background(), fixture.installation, CreateRequest{Output: filepath.Join(t.TempDir(), "drained.zip"), Drain: true}, dependencies); err != nil {
t.Fatal(err)
}
if sleeps == 0 || runner.streamWhileRunning {
t.Fatalf("drain did not wait for a stopped snapshot: sleeps=%d streamedWhileRunning=%v", sleeps, runner.streamWhileRunning)
}
}
func TestCreateDefaultPathUsesHomeInstallationIDUTCAndSourceRevision(t *testing.T) {
fixture := newBackupFixture(t, "local")
runner := newBackupRunner(fixture.installation, false)
dependencies := testDependencies(t, runner)
home := t.TempDir()
dependencies.homeDir = func() (string, error) { return home, nil }
result, err := createWithDependencies(context.Background(), fixture.installation, CreateRequest{}, dependencies)
if err != nil {
t.Fatal(err)
}
wantDirectory := filepath.Join(home, ".thothii", "backups", fixture.installationID)
if filepath.Dir(result.Path) != wantDirectory {
t.Fatalf("default directory = %q, want %q", filepath.Dir(result.Path), wantDirectory)
}
name := filepath.Base(result.Path)
for _, fragment := range []string{"20260816T081112Z", testRevision} {
if !strings.Contains(name, fragment) {
t.Errorf("default archive name %q does not contain %q", name, fragment)
}
}
}
func TestCreateCleansIncompleteArchiveAndRestoresRunningStateAfterStreamFailure(t *testing.T) {
fixture := newBackupFixture(t, "local")
runner := newBackupRunner(fixture.installation, true)
runner.failStreamAt = 3
directory := t.TempDir()
output := filepath.Join(directory, "failed.zip")
if _, err := createWithDependencies(context.Background(), fixture.installation, CreateRequest{Output: output}, testDependencies(t, runner)); err == nil {
t.Fatal("Create() succeeded after a volume stream failure")
}
if !runner.running || !runner.coreRunning || runner.maintenance {
t.Fatalf("running state was not recovered: running=%v core=%v maintenance=%v", runner.running, runner.coreRunning, runner.maintenance)
}
assertNoBackupArtifacts(t, directory)
}
func TestCreateCleansIncompleteArchiveWhenVolumeHelperReportsANonzeroExit(t *testing.T) {
fixture := newBackupFixture(t, "local")
runner := newBackupRunner(fixture.installation, false)
runner.exitOnlyAt = 3
directory := t.TempDir()
if _, err := createWithDependencies(context.Background(), fixture.installation, CreateRequest{Output: filepath.Join(directory, "failed.zip")}, testDependencies(t, runner)); err == nil {
t.Fatal("Create() succeeded after a nonzero volume-helper exit")
}
assertNoBackupArtifacts(t, directory)
}
func TestCreateCleansIncompleteArchiveAfterAtomicPublishFailure(t *testing.T) {
fixture := newBackupFixture(t, "local")
runner := newBackupRunner(fixture.installation, false)
directory := t.TempDir()
dependencies := testDependencies(t, runner)
dependencies.rename = func(string, string) error { return errors.New("publish failed") }
if _, err := createWithDependencies(context.Background(), fixture.installation, CreateRequest{Output: filepath.Join(directory, "failed.zip")}, dependencies); err == nil {
t.Fatal("Create() succeeded after publish failure")
}
assertNoBackupArtifacts(t, directory)
}
func TestCreateExcludesExternalSecretPayloadsByDefaultButRecordsDigests(t *testing.T) {
fixture := newBackupFixture(t, "local")
runner := newBackupRunner(fixture.installation, false)
output := filepath.Join(t.TempDir(), "no-external-secrets.zip")
if _, err := createWithDependencies(context.Background(), fixture.installation, CreateRequest{Output: output}, testDependencies(t, runner)); err != nil {
t.Fatal(err)
}
archive := readFixtureArchive(t, output)
all := bytes.Join(mapValues(archive.files), nil)
if bytes.Contains(all, []byte(fixture.secretValue)) {
t.Fatal("default archive contains an external secret value")
}
secretReferences := 0
workspaceSecrets := false
for _, entry := range archive.manifest.Entries {
if entry.Kind == EntrySecretReference {
secretReferences++
if entry.Archived || entry.SourcePath == "" || entry.SHA256 != DigestBytes([]byte(fixture.secretValue)) {
t.Fatalf("secret reference = %#v", entry)
}
}
if entry.Path == "volumes/workspace-secrets.tar" {
workspaceSecrets = entry.Archived
}
}
if secretReferences != 1 || !workspaceSecrets || archive.manifest.IncludesSecrets {
t.Fatalf("secret behavior: references=%d workspace=%v includes=%v", secretReferences, workspaceSecrets, archive.manifest.IncludesSecrets)
}
if strings.Contains(strings.Join(runner.calls, "\n"), fixture.secretValue) {
t.Fatal("external secret value appeared in a process argument")
}
}
func TestCreateRequiresConfirmationToIncludeSecretsAndUsesOwnerOnlyMode(t *testing.T) {
fixture := newBackupFixture(t, "local")
output := filepath.Join(t.TempDir(), "with-secrets.zip")
request := CreateRequest{Output: output, IncludeSecrets: true}
if _, err := createWithDependencies(context.Background(), fixture.installation, request, testDependencies(t, newBackupRunner(fixture.installation, false))); !errors.Is(err, ErrConfirmationRequired) {
t.Fatalf("Create() error = %v, want ErrConfirmationRequired", err)
}
request.Confirm = true
result, err := createWithDependencies(context.Background(), fixture.installation, request, testDependencies(t, newBackupRunner(fixture.installation, false)))
if err != nil {
t.Fatal(err)
}
if result.Warning == "" || strings.Contains(result.Warning, fixture.secretValue) {
t.Fatalf("custody warning = %q", result.Warning)
}
info, err := os.Stat(output)
if err != nil {
t.Fatal(err)
}
if got := info.Mode().Perm(); got != 0o600 {
t.Fatalf("archive mode = %#o, want 0600", got)
}
archive := readFixtureArchive(t, output)
if !archive.manifest.IncludesSecrets || !bytes.Contains(bytes.Join(mapValues(archive.files), nil), []byte(fixture.secretValue)) {
t.Fatal("confirmed archive does not include the external secret payload")
}
}
func TestCreateRejectsInlineSecretValuesBeforeWritingAnArchive(t *testing.T) {
fixture := newBackupFixture(t, "local")
fixture.environment = append(fixture.environment, "THT_LLM_API_KEY=must-never-enter-an-archive")
fixture.writeEnvironment(t)
directory := t.TempDir()
runner := newBackupRunner(fixture.installation, false)
_, err := createWithDependencies(context.Background(), fixture.installation, CreateRequest{Output: filepath.Join(directory, "unsafe.zip")}, testDependencies(t, runner))
if err == nil || !strings.Contains(err.Error(), "inline secret") || strings.Contains(err.Error(), "must-never") {
t.Fatalf("Create() error = %v, want value-free inline-secret refusal", err)
}
if len(runner.calls) != 0 {
t.Fatalf("Docker was called before unsafe environment refusal: %v", runner.calls)
}
assertNoBackupArtifacts(t, directory)
}
func TestCreateRejectsInlineAuthorizationValuesBeforeWritingAnArchive(t *testing.T) {
fixture := newBackupFixture(t, "local")
fixture.environment = append(fixture.environment, "DWH_AUTHORIZATION=must-never-enter-an-archive")
fixture.writeEnvironment(t)
directory := t.TempDir()
runner := newBackupRunner(fixture.installation, false)
_, err := createWithDependencies(context.Background(), fixture.installation, CreateRequest{Output: filepath.Join(directory, "unsafe.zip")}, testDependencies(t, runner))
if err == nil || !strings.Contains(err.Error(), "inline secret") || strings.Contains(err.Error(), "must-never") {
t.Fatalf("Create() error = %v, want value-free inline-secret refusal", err)
}
if len(runner.calls) != 0 {
t.Fatalf("Docker was called before unsafe environment refusal: %v", runner.calls)
}
assertNoBackupArtifacts(t, directory)
}
func TestCreateIncludesServerPreservationRootsWithoutRecursingIntoBackupRoot(t *testing.T) {
fixture := newBackupFixture(t, "server")
for _, item := range []struct{ variable, name, value string }{
{"THT_DATA_ROOT", "data", "session-state"},
{"THT_PI_STATE_ROOT", "pi-state", "pi-state"},
{"THT_WORKSPACE_REGISTRY_ROOT", "registry", "workspace-registry"},
} {
root := filepath.Join(fixture.root, item.name)
if err := os.MkdirAll(root, 0o700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(root, "payload"), []byte(item.value), 0o600); err != nil {
t.Fatal(err)
}
fixture.environment = append(fixture.environment, item.variable+"="+root)
}
backupRoot := filepath.Join(fixture.root, "existing-backups")
if err := os.MkdirAll(backupRoot, 0o700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(backupRoot, "old-secret-backup"), []byte("must-not-be-recursed"), 0o600); err != nil {
t.Fatal(err)
}
fixture.environment = append(fixture.environment, "THT_BACKUP_ROOT="+backupRoot)
fixture.writeEnvironment(t)
output := filepath.Join(t.TempDir(), "server.zip")
if _, err := createWithDependencies(context.Background(), fixture.installation, CreateRequest{Output: output}, testDependencies(t, newBackupRunner(fixture.installation, false))); err != nil {
t.Fatal(err)
}
archive := readFixtureArchive(t, output)
all := bytes.Join(mapValues(archive.files), nil)
for _, value := range []string{"session-state", "pi-state", "workspace-registry"} {
if !bytes.Contains(all, []byte(value)) {
t.Errorf("server preservation payload %q is missing", value)
}
}
if bytes.Contains(all, []byte("must-not-be-recursed")) {
t.Fatal("backup destination root was recursively included")
}
}
func TestCreateHonorsTheSharedInstallationLifecycleLock(t *testing.T) {
fixture := newBackupFixture(t, "local")
lock, err := lifecycle.Acquire(fixture.installation)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = lock.Release() })
runner := newBackupRunner(fixture.installation, false)
_, err = createWithDependencies(context.Background(), fixture.installation, CreateRequest{Output: filepath.Join(t.TempDir(), "locked.zip")}, testDependencies(t, runner))
if !errors.Is(err, lifecycle.ErrLocked) {
t.Fatalf("Create() error = %v, want lifecycle.ErrLocked", err)
}
if len(runner.calls) != 0 {
t.Fatalf("Docker runner was called while lock was held: %v", runner.calls)
}
}
type backupFixture struct {
root string
installationID string
installation config.Installation
environment []string
secretValue string
}
func newBackupFixture(t *testing.T, profile string) *backupFixture {
t.Helper()
temporaryRoot, err := filepath.EvalSymlinks(os.TempDir())
if err != nil {
t.Fatal(err)
}
root, err := os.MkdirTemp(temporaryRoot, "tht-backup-")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = os.RemoveAll(root) })
for _, directory := range []string{filepath.Join(root, "deploy", "pi"), filepath.Join(root, "deploy", "fixture")} {
if err := os.MkdirAll(directory, 0o700); err != nil {
t.Fatal(err)
}
}
write := func(path, contents string) {
if err := os.WriteFile(path, []byte(contents), 0o600); err != nil {
t.Fatal(err)
}
}
write(filepath.Join(root, "compose.yaml"), "services: {}\n")
write(filepath.Join(root, "deploy", "compose."+profile+".yaml"), "services: {}\n")
write(filepath.Join(root, "deploy", "pi", "models.json"), `{"providers":{}}`)
write(filepath.Join(root, "deploy", "pi", "settings.json"), `{"enabledModels":[]}`)
override := filepath.Join(root, "deploy", "fixture", "extra.yaml")
write(override, "services: {}\n")
installationID := profile + "-fixture"
descriptorDirectory := filepath.Join(root, "deploy", installationID)
if err := os.MkdirAll(descriptorDirectory, 0o700); err != nil {
t.Fatal(err)
}
descriptor := filepath.Join(descriptorDirectory, "thothii-installation.yaml")
environment := filepath.Join(descriptorDirectory, "operator.env")
write(descriptor, "profile: "+profile+"\nprojectDirectory: "+root+"\nenvFile: "+environment+"\n")
secretValue := "external-secret-value-for-backup-test"
secretDirectory, err := os.MkdirTemp(temporaryRoot, "tht-backup-secret-")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = os.RemoveAll(secretDirectory) })
secretPath := filepath.Join(secretDirectory, "external-secret")
write(secretPath, secretValue)
fixture := &backupFixture{
root: root,
installationID: installationID,
secretValue: secretValue,
environment: []string{
"THT_WORKSPACE_INSTALLATION_ID=" + installationID,
"THT_SECRETS_FILE=" + secretPath,
"THT_WORKSPACE_GIT_REMOTE=https://git.example.invalid/workspaces.git",
"THT_WORKSPACE_GIT_BRANCH=main",
},
installation: config.Installation{
Path: descriptor, Profile: profile, ProjectDirectory: root, EnvFile: environment,
Overrides: []string{override},
},
}
fixture.writeEnvironment(t)
currentImage := fixture.installation.CurrentImageOverridePath()
if err := os.MkdirAll(filepath.Dir(currentImage), 0o700); err != nil {
t.Fatal(err)
}
write(currentImage, "services:\n core:\n image: core:current\n")
return fixture
}
func (fixture *backupFixture) writeEnvironment(t *testing.T) {
t.Helper()
if err := os.WriteFile(fixture.installation.EnvFile, []byte(strings.Join(fixture.environment, "\n")+"\n"), 0o600); err != nil {
t.Fatal(err)
}
}
type fakeBackupRunner struct {
installation config.Installation
running bool
coreRunning bool
maintenance bool
sessionResponses []string
calls []string
streams int
failStreamAt int
exitOnlyAt int
streamWhileRunning bool
stopCount int
startCount int
healthChecks int
}
func newBackupRunner(installation config.Installation, running bool) *fakeBackupRunner {
return &fakeBackupRunner{installation: installation, running: running, coreRunning: running}
}
func (runner *fakeBackupRunner) Run(_ context.Context, args []string, _ io.Reader) (compose.Result, error) {
command := strings.Join(args, " ")
runner.calls = append(runner.calls, command)
switch {
case strings.Contains(command, " config --format json"):
volumes := map[string]map[string]string{}
for _, logical := range requiredTestVolumes {
volumes[logical] = map[string]string{"name": runner.installation.ProjectName() + "_" + logical}
}
payload := map[string]any{
"volumes": volumes,
"services": map[string]any{
"core": map[string]any{"image": "thothii-core:test"},
"frontend": map[string]any{"image": "thothii-frontend:test"},
},
}
encoded, _ := json.Marshal(payload)
return compose.Result{Stdout: string(encoded)}, nil
case strings.HasPrefix(command, "volume inspect "):
items := make([]map[string]any, 0, len(requiredTestVolumes))
for _, logical := range requiredTestVolumes {
items = append(items, map[string]any{
"Name": runner.installation.ProjectName() + "_" + logical,
"Driver": "local",
"Labels": map[string]string{"com.docker.compose.project": runner.installation.ProjectName(), "com.docker.compose.volume": logical},
})
}
encoded, _ := json.Marshal(items)
return compose.Result{Stdout: string(encoded)}, nil
case strings.Contains(command, " images --format json"):
return compose.Result{Stdout: "{\"Service\":\"core\",\"Repository\":\"thothii-core\",\"Tag\":\"test\",\"ID\":\"sha256:core\"}\n{\"Service\":\"frontend\",\"Repository\":\"thothii-frontend\",\"Tag\":\"test\",\"ID\":\"sha256:frontend\"}\n"}, nil
case strings.Contains(command, " ps --status running -q core"):
if runner.coreRunning {
return compose.Result{Stdout: "core-id\n"}, nil
}
return compose.Result{}, nil
case strings.Contains(command, " ps --status running -q"):
if runner.running {
return compose.Result{Stdout: "container-id\n"}, nil
}
return compose.Result{}, nil
case strings.Contains(command, "/internal/maintenance/status"):
return compose.Result{Stdout: fmt.Sprintf(`{"active":%t,"admissions":0,"recoveryRequired":false}`, runner.maintenance)}, nil
case strings.Contains(command, "/internal/maintenance/activate"):
runner.maintenance = true
return compose.Result{Stdout: `{"active":true,"admissions":0,"recoveryRequired":false}`}, nil
case strings.Contains(command, "/internal/maintenance/deactivate"):
runner.maintenance = false
return compose.Result{Stdout: `{"active":false,"admissions":0,"recoveryRequired":false}`}, nil
case strings.Contains(command, "/sessions?scope="):
if len(runner.sessionResponses) == 0 {
return compose.Result{Stdout: `[]`}, nil
}
response := runner.sessionResponses[0]
if len(runner.sessionResponses) > 1 {
runner.sessionResponses = runner.sessionResponses[1:]
}
return compose.Result{Stdout: response}, nil
case strings.HasSuffix(command, " stop"):
runner.stopCount++
runner.running, runner.coreRunning = false, false
return compose.Result{}, nil
case strings.HasSuffix(command, " start"):
runner.startCount++
runner.running, runner.coreRunning = true, true
return compose.Result{}, nil
case strings.Contains(command, " ps --all --format json"):
runner.healthChecks++
return compose.Result{Stdout: healthyServicesPayload()}, nil
default:
return compose.Result{}, fmt.Errorf("unexpected fake Docker command: %s", command)
}
}
func (runner *fakeBackupRunner) Stream(_ context.Context, args []string, _ io.Reader, stdout io.Writer) (compose.Result, error) {
runner.calls = append(runner.calls, strings.Join(args, " "))
runner.streams++
if runner.running {
runner.streamWhileRunning = true
}
writer := tar.NewWriter(stdout)
payload := []byte(fmt.Sprintf("volume-%d", runner.streams))
if err := writer.WriteHeader(&tar.Header{Name: "payload", Mode: 0o600, Size: int64(len(payload))}); err != nil {
return compose.Result{}, err
}
if _, err := writer.Write(payload); err != nil {
return compose.Result{}, err
}
if err := writer.Close(); err != nil {
return compose.Result{}, err
}
if runner.failStreamAt == runner.streams {
return compose.Result{ExitCode: 1}, errors.New("fixture volume stream failed")
}
if runner.exitOnlyAt == runner.streams {
return compose.Result{ExitCode: 1}, nil
}
return compose.Result{}, nil
}
func (runner *fakeBackupRunner) SessionInventoryScope() string {
if runner.installation.Profile == "local" {
return "mine"
}
return "all"
}
func testDependencies(t *testing.T, runner archiveRunner) dependencies {
t.Helper()
return dependencies{
runner: runner,
now: func() time.Time { return time.Date(2026, 8, 16, 8, 11, 12, 0, time.UTC) },
homeDir: func() (string, error) { return t.TempDir(), nil },
revision: func(context.Context, string) (string, error) { return testRevision, nil },
sleep: func(time.Duration) {},
rename: os.Rename,
}
}
type fixtureArchive struct {
order []string
files map[string][]byte
manifest Manifest
}
func readFixtureArchive(t *testing.T, path string) fixtureArchive {
t.Helper()
reader, err := zip.OpenReader(path)
if err != nil {
t.Fatal(err)
}
defer reader.Close()
result := fixtureArchive{files: make(map[string][]byte)}
for _, file := range reader.File {
result.order = append(result.order, file.Name)
opened, err := file.Open()
if err != nil {
t.Fatal(err)
}
contents, err := io.ReadAll(opened)
closeErr := opened.Close()
if err != nil || closeErr != nil {
t.Fatalf("read %s: %v / %v", file.Name, err, closeErr)
}
result.files[file.Name] = contents
}
if err := json.Unmarshal(result.files[ManifestPath], &result.manifest); err != nil {
t.Fatalf("decode manifest: %v", err)
}
return result
}
func mapValues(values map[string][]byte) [][]byte {
keys := make([]string, 0, len(values))
for key := range values {
keys = append(keys, key)
}
sort.Strings(keys)
result := make([][]byte, 0, len(keys))
for _, key := range keys {
result = append(result, values[key])
}
return result
}
func activeSessionPayload() string {
return `[{"status":"running","archived":false}]`
}
func healthyServicesPayload() string {
return strings.Join([]string{
`{"Service":"core","State":"running","Health":"healthy"}`,
`{"Service":"frontend","State":"running","Health":"healthy"}`,
`{"Service":"qdrant","State":"running","Health":"healthy"}`,
`{"Service":"embedding","State":"running","Health":"healthy"}`,
`{"Service":"embedding-model-init","State":"exited","ExitCode":0}`,
}, "\n")
}
func assertNoBackupArtifacts(t *testing.T, directory string) {
t.Helper()
entries, err := os.ReadDir(directory)
if err != nil {
t.Fatal(err)
}
if len(entries) != 0 {
names := make([]string, 0, len(entries))
for _, entry := range entries {
names = append(names, entry.Name())
}
t.Fatalf("incomplete backup artifacts remain: %v", names)
}
}
tools/tht/internal/backup/manifest.go
+215
View File
@@ -0,0 +1,215 @@
package backup
import (
"bytes"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"path"
"regexp"
"sort"
"strings"
"time"
)
const (
CurrentSchemaVersion = 1
ManifestPath = "manifest.json"
EntryFile = "file"
EntryVolume = "volume"
EntrySecretReference = "external-secret-reference"
EntryExternalSecret = "external-secret"
EntryPreservationReference = "preservation-root-reference"
)
var (
checksumPattern = regexp.MustCompile(`^sha256:[0-9a-f]{64}$`)
revisionPattern = regexp.MustCompile(`^[0-9a-f]{40}([0-9a-f]{24})?$`)
)
// Entry describes one logical backup payload or one external prerequisite.
type Entry struct {
Path string `json:"path"`
Kind string `json:"kind"`
Owner string `json:"owner"`
LogicalName string `json:"logical_name,omitempty"`
SourcePath string `json:"source_path,omitempty"`
SHA256 string `json:"sha256"`
Size int64 `json:"size"`
Mode uint32 `json:"mode,omitempty"`
Archived bool `json:"archived"`
Sensitive bool `json:"sensitive,omitempty"`
}
// ImageIdentity records the configured image reference and, when available, the local image ID.
type ImageIdentity struct {
Service string `json:"service"`
Reference string `json:"reference"`
ID string `json:"id,omitempty"`
}
// VolumeMetadata binds a logical Compose volume to its installation-owned Docker identity.
type VolumeMetadata struct {
LogicalName string `json:"logical_name"`
Name string `json:"name"`
Driver string `json:"driver"`
Labels map[string]string `json:"labels,omitempty"`
}
// Manifest is the versioned restore contract written as the last archive member.
type Manifest struct {
SchemaVersion int `json:"schema_version"`
InstallationID string `json:"installation_id"`
CreatedAt time.Time `json:"created_at"`
SourceRevision string `json:"source_revision"`
IncludesSecrets bool `json:"includes_secrets"`
ComposeProject string `json:"compose_project"`
Images []ImageIdentity `json:"images"`
Volumes []VolumeMetadata `json:"volumes"`
Entries []Entry `json:"entries"`
}
// DigestBytes returns the manifest's canonical checksum representation.
func DigestBytes(value []byte) string {
digest := sha256.Sum256(value)
return "sha256:" + hex.EncodeToString(digest[:])
}
// Finalize normalizes archive paths, orders every repeated field and validates the schema.
func (manifest *Manifest) Finalize() error {
manifest.CreatedAt = manifest.CreatedAt.UTC()
for index := range manifest.Entries {
normalized, err := normalizeArchivePath(manifest.Entries[index].Path)
if err != nil {
return err
}
manifest.Entries[index].Path = normalized
}
sort.Slice(manifest.Entries, func(left, right int) bool {
if manifest.Entries[left].Path != manifest.Entries[right].Path {
return manifest.Entries[left].Path < manifest.Entries[right].Path
}
if manifest.Entries[left].Kind != manifest.Entries[right].Kind {
return manifest.Entries[left].Kind < manifest.Entries[right].Kind
}
return manifest.Entries[left].Owner < manifest.Entries[right].Owner
})
sort.Slice(manifest.Images, func(left, right int) bool { return manifest.Images[left].Service < manifest.Images[right].Service })
sort.Slice(manifest.Volumes, func(left, right int) bool {
return manifest.Volumes[left].LogicalName < manifest.Volumes[right].LogicalName
})
return manifest.Validate()
}
// Validate rejects unsupported or unsafe restore contracts.
func (manifest Manifest) Validate() error {
if manifest.SchemaVersion != CurrentSchemaVersion {
return fmt.Errorf("unsupported backup manifest schema version %d", manifest.SchemaVersion)
}
if strings.TrimSpace(manifest.InstallationID) == "" || strings.ContainsAny(manifest.InstallationID, `/\\`) {
return errors.New("backup manifest installation ID is invalid")
}
if manifest.CreatedAt.IsZero() || manifest.CreatedAt.Location() != time.UTC {
return errors.New("backup manifest creation time must be UTC")
}
if !revisionPattern.MatchString(manifest.SourceRevision) {
return errors.New("backup manifest source revision is invalid")
}
if strings.TrimSpace(manifest.ComposeProject) == "" {
return errors.New("backup manifest Compose project is missing")
}
seenPaths := make(map[string]struct{}, len(manifest.Entries))
includedExternalSecrets := 0
for _, entry := range manifest.Entries {
if _, err := normalizeArchivePath(entry.Path); err != nil {
return err
}
if _, exists := seenPaths[entry.Path]; exists {
return fmt.Errorf("backup manifest contains duplicate entry path %q", entry.Path)
}
seenPaths[entry.Path] = struct{}{}
if entry.Owner == "" || entry.Kind == "" || entry.Size < 0 || !checksumPattern.MatchString(entry.SHA256) {
return fmt.Errorf("backup manifest entry %q is invalid", entry.Path)
}
if (entry.Kind == EntrySecretReference || entry.Kind == EntryExternalSecret) && entry.SourcePath == "" {
return fmt.Errorf("backup manifest external secret entry %q has no source path", entry.Path)
}
if entry.Kind == EntryExternalSecret {
if !entry.Archived || !entry.Sensitive {
return fmt.Errorf("backup manifest external secret entry %q is not marked sensitive and archived", entry.Path)
}
includedExternalSecrets++
}
}
if manifest.IncludesSecrets != (includedExternalSecrets > 0) {
return errors.New("backup manifest external secret marker does not match included external secret payloads")
}
seenImages := make(map[string]struct{}, len(manifest.Images))
for _, image := range manifest.Images {
if image.Service == "" || image.Reference == "" {
return errors.New("backup manifest image identity is incomplete")
}
if _, exists := seenImages[image.Service]; exists {
return fmt.Errorf("backup manifest contains duplicate image service %q", image.Service)
}
seenImages[image.Service] = struct{}{}
}
seenVolumes := make(map[string]struct{}, len(manifest.Volumes))
for _, volume := range manifest.Volumes {
if volume.LogicalName == "" || volume.Name == "" || volume.Driver == "" {
return errors.New("backup manifest volume metadata is incomplete")
}
if _, exists := seenVolumes[volume.LogicalName]; exists {
return fmt.Errorf("backup manifest contains duplicate volume %q", volume.LogicalName)
}
seenVolumes[volume.LogicalName] = struct{}{}
}
return nil
}
// JSON returns a deterministic, indented representation after validating a copy.
func (manifest Manifest) JSON() ([]byte, error) {
manifest.Entries = append([]Entry(nil), manifest.Entries...)
manifest.Images = append([]ImageIdentity(nil), manifest.Images...)
manifest.Volumes = append([]VolumeMetadata(nil), manifest.Volumes...)
if err := manifest.Finalize(); err != nil {
return nil, err
}
value, err := json.MarshalIndent(manifest, "", " ")
if err != nil {
return nil, err
}
return append(value, '\n'), nil
}
// DecodeManifest decodes exactly one supported manifest document.
func DecodeManifest(value []byte) (Manifest, error) {
decoder := json.NewDecoder(bytes.NewReader(value))
decoder.DisallowUnknownFields()
var manifest Manifest
if err := decoder.Decode(&manifest); err != nil {
return Manifest{}, fmt.Errorf("decode backup manifest: %w", err)
}
if decoder.More() {
return Manifest{}, errors.New("backup manifest contains trailing data")
}
if err := manifest.Finalize(); err != nil {
return Manifest{}, err
}
return manifest, nil
}
func normalizeArchivePath(value string) (string, error) {
value = strings.ReplaceAll(value, `\`, "/")
if value == "" || strings.HasPrefix(value, "/") {
return "", errors.New("backup manifest entry path is empty or absolute")
}
normalized := path.Clean(value)
if normalized == "." || normalized == ".." || strings.HasPrefix(normalized, "../") {
return "", fmt.Errorf("backup manifest entry path %q escapes the archive", value)
}
return normalized, nil
}
tools/tht/internal/backup/manifest_test.go
+131
View File
@@ -0,0 +1,131 @@
package backup
import (
"encoding/json"
"strings"
"testing"
"time"
)
const testRevision = "0123456789abcdef0123456789abcdef01234567"
func TestManifestFinalizationNormalizesAndOrdersEntriesDeterministically(t *testing.T) {
manifest := Manifest{
SchemaVersion: CurrentSchemaVersion,
InstallationID: "local-dev",
CreatedAt: time.Date(2026, 8, 16, 10, 11, 12, 123, time.FixedZone("test", 2*60*60)),
SourceRevision: testRevision,
ComposeProject: "thothii-test",
Entries: []Entry{
{Path: `volumes\sessions.tar`, Kind: EntryVolume, Owner: "volume:sessions", SHA256: DigestBytes([]byte("sessions")), Size: 8, Archived: true},
{Path: "configuration/pi/models.json", Kind: EntryFile, Owner: "pi", SHA256: DigestBytes([]byte("models")), Size: 6, Archived: true},
},
Images: []ImageIdentity{
{Service: "frontend", Reference: "frontend:test", ID: "sha256:front"},
{Service: "core", Reference: "core:test", ID: "sha256:core"},
},
Volumes: []VolumeMetadata{
{LogicalName: "sessions", Name: "thothii-test_sessions", Driver: "local"},
{LogicalName: "pi-state", Name: "thothii-test_pi-state", Driver: "local"},
},
}
if err := manifest.Finalize(); err != nil {
t.Fatal(err)
}
if got, want := manifest.CreatedAt, time.Date(2026, 8, 16, 8, 11, 12, 123, time.UTC); !got.Equal(want) || got.Location() != time.UTC {
t.Fatalf("CreatedAt = %v (%v), want %v UTC", got, got.Location(), want)
}
if got := []string{manifest.Entries[0].Path, manifest.Entries[1].Path}; got[0] != "configuration/pi/models.json" || got[1] != "volumes/sessions.tar" {
t.Fatalf("entry order = %v", got)
}
if manifest.Images[0].Service != "core" || manifest.Volumes[0].LogicalName != "pi-state" {
t.Fatalf("metadata was not deterministically ordered: images=%v volumes=%v", manifest.Images, manifest.Volumes)
}
first, err := manifest.JSON()
if err != nil {
t.Fatal(err)
}
second, err := manifest.JSON()
if err != nil {
t.Fatal(err)
}
if string(first) != string(second) {
t.Fatalf("manifest encoding is not deterministic:\n%s\n%s", first, second)
}
}
func TestManifestRejectsUnsafePathsInvalidChecksumsAndUnsupportedSchemas(t *testing.T) {
valid := Manifest{
SchemaVersion: CurrentSchemaVersion,
InstallationID: "local-dev",
CreatedAt: time.Now().UTC(),
SourceRevision: testRevision,
ComposeProject: "thothii-test",
Entries: []Entry{{Path: "configuration/operator.env", Kind: EntryFile, Owner: "installation", SHA256: DigestBytes(nil), Archived: true}},
}
for _, mutate := range []func(*Manifest){
func(value *Manifest) { value.SchemaVersion = CurrentSchemaVersion + 1 },
func(value *Manifest) { value.Entries[0].Path = "../outside" },
func(value *Manifest) { value.Entries[0].Path = "/absolute" },
func(value *Manifest) { value.Entries[0].SHA256 = "sha256:not-a-digest" },
func(value *Manifest) { value.Entries = append(value.Entries, value.Entries[0]) },
} {
candidate := valid
candidate.Entries = append([]Entry(nil), valid.Entries...)
mutate(&candidate)
if err := candidate.Finalize(); err == nil {
t.Fatalf("Finalize() accepted invalid manifest: %#v", candidate)
}
}
}
func TestManifestSecretMarkerRequiresIncludedExternalSecretPayload(t *testing.T) {
base := Manifest{
SchemaVersion: CurrentSchemaVersion,
InstallationID: "local-dev",
CreatedAt: time.Now().UTC(),
SourceRevision: testRevision,
ComposeProject: "thothii-test",
Entries: []Entry{
{Path: "volumes/workspace-secrets.tar", Kind: EntryVolume, Owner: "volume:workspace-secrets", SHA256: DigestBytes([]byte("managed")), Size: 7, Archived: true, Sensitive: true},
{Path: "external-secrets/000", Kind: EntrySecretReference, Owner: "external-secret", SHA256: DigestBytes([]byte("external")), Size: 8, Archived: false, Sensitive: true, SourcePath: "/protected/secret"},
},
}
if err := base.Finalize(); err != nil {
t.Fatalf("managed workspace secrets and an excluded external reference must be valid: %v", err)
}
base.IncludesSecrets = true
if err := base.Finalize(); err == nil || !strings.Contains(err.Error(), "external secret") {
t.Fatalf("Finalize() error = %v, want missing included external secret", err)
}
for index := range base.Entries {
if base.Entries[index].Kind == EntrySecretReference {
base.Entries[index].Kind = EntryExternalSecret
base.Entries[index].Archived = true
base.Entries[index].Path = "external-secrets/000-secret"
}
}
if err := base.Finalize(); err != nil {
t.Fatalf("included external secret marker was rejected: %v", err)
}
}
func TestDecodeManifestRejectsASecondDocument(t *testing.T) {
manifest := Manifest{
SchemaVersion: CurrentSchemaVersion,
InstallationID: "local-dev",
CreatedAt: time.Date(2026, 8, 16, 8, 11, 12, 0, time.UTC),
SourceRevision: testRevision,
ComposeProject: "thothii-test",
}
encoded, err := json.Marshal(manifest)
if err != nil {
t.Fatal(err)
}
if _, err := DecodeManifest(append(encoded, encoded...)); err == nil {
t.Fatal("DecodeManifest() accepted a second JSON document")
}
}
tools/tht/internal/lifecycle/lock.go
+113
View File
@@ -0,0 +1,113 @@
// Package lifecycle coordinates installation-wide mutating operations.
package lifecycle
import (
"crypto/rand"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"sync"
"time"
"github.com/aritmolab/thothii/tools/tht/internal/config"
)
var (
ErrLocked = errors.New("another lifecycle operation is already running for this installation")
ErrOwnership = errors.New("lifecycle lock ownership changed; refusing to remove it")
)
const lockFileName = "lifecycle.lock.owner.json"
type owner struct {
Token string `json:"token"`
PID int `json:"pid"`
CreatedAt time.Time `json:"created_at"`
}
// Lock is an exclusively-created owner file scoped to one installation descriptor.
type Lock struct {
path string
token string
mu sync.Mutex
released bool
}
// Acquire obtains the shared lock used by backup, restore, Pi lifecycle and product updates.
func Acquire(installation config.Installation) (*Lock, error) {
directory := installation.ControlDirectory()
if err := os.MkdirAll(directory, 0o700); err != nil {
return nil, fmt.Errorf("create lifecycle control directory: %w", err)
}
info, err := os.Lstat(directory)
if err != nil || !info.IsDir() || info.Mode()&os.ModeSymlink != 0 {
return nil, errors.New("lifecycle control directory is not a regular directory")
}
if err := os.Chmod(directory, 0o700); err != nil {
return nil, fmt.Errorf("protect lifecycle control directory: %w", err)
}
tokenBytes := make([]byte, 16)
if _, err := rand.Read(tokenBytes); err != nil {
return nil, fmt.Errorf("create lifecycle lock owner: %w", err)
}
token := hex.EncodeToString(tokenBytes)
path := filepath.Join(directory, lockFileName)
file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
if errors.Is(err, os.ErrExist) {
return nil, ErrLocked
}
if err != nil {
return nil, fmt.Errorf("acquire lifecycle lock: %w", err)
}
value := owner{Token: token, PID: os.Getpid(), CreatedAt: time.Now().UTC()}
encodeErr := json.NewEncoder(file).Encode(value)
if encodeErr == nil {
encodeErr = file.Sync()
}
closeErr := file.Close()
if encodeErr != nil || closeErr != nil {
_ = os.Remove(path)
return nil, fmt.Errorf("persist lifecycle lock owner: %w", errors.Join(encodeErr, closeErr))
}
return &Lock{path: path, token: token}, nil
}
// Path returns the installation-private owner-file path for diagnostics and tests.
func (lock *Lock) Path() string {
if lock == nil {
return ""
}
return lock.path
}
// Release removes only the owner file created by this Lock.
func (lock *Lock) Release() error {
if lock == nil {
return nil
}
lock.mu.Lock()
defer lock.mu.Unlock()
if lock.released {
return nil
}
contents, err := os.ReadFile(lock.path)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
return ErrOwnership
}
return fmt.Errorf("read lifecycle lock owner: %w", err)
}
var current owner
if json.Unmarshal(contents, &current) != nil || current.Token == "" || current.Token != lock.token {
return ErrOwnership
}
if err := os.Remove(lock.path); err != nil {
return fmt.Errorf("release lifecycle lock: %w", err)
}
lock.released = true
return nil
}
tools/tht/internal/lifecycle/lock_test.go
+50
View File
@@ -0,0 +1,50 @@
package lifecycle
import (
"errors"
"os"
"path/filepath"
"testing"
"github.com/aritmolab/thothii/tools/tht/internal/config"
)
func TestLifecycleLockIsExclusivePerInstallationAndReusableAfterRelease(t *testing.T) {
installation := config.Installation{ProjectDirectory: t.TempDir(), Path: filepath.Join(t.TempDir(), "thothii-installation.yaml")}
first, err := Acquire(installation)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = first.Release() })
if _, err := Acquire(installation); !errors.Is(err, ErrLocked) {
t.Fatalf("second Acquire() error = %v, want ErrLocked", err)
}
if err := first.Release(); err != nil {
t.Fatal(err)
}
second, err := Acquire(installation)
if err != nil {
t.Fatalf("Acquire() after release: %v", err)
}
if err := second.Release(); err != nil {
t.Fatal(err)
}
}
func TestLifecycleLockReleaseDoesNotRemoveAnotherOwnersFile(t *testing.T) {
installation := config.Installation{ProjectDirectory: t.TempDir(), Path: filepath.Join(t.TempDir(), "thothii-installation.yaml")}
lock, err := Acquire(installation)
if err != nil {
t.Fatal(err)
}
if err := os.WriteFile(lock.Path(), []byte(`{"token":"different-owner"}`), 0o600); err != nil {
t.Fatal(err)
}
if err := lock.Release(); !errors.Is(err, ErrOwnership) {
t.Fatalf("Release() error = %v, want ErrOwnership", err)
}
if _, err := os.Stat(lock.Path()); err != nil {
t.Fatalf("foreign lock was removed: %v", err)
}
}