diff --git a/tools/tht/cmd/tht/main.go b/tools/tht/cmd/tht/main.go index 4086a2c8..de80d5ca 100644 --- a/tools/tht/cmd/tht/main.go +++ b/tools/tht/cmd/tht/main.go @@ -14,9 +14,11 @@ import ( "strconv" "strings" + "github.com/aritmolab/thothii/tools/tht/internal/backup" "github.com/aritmolab/thothii/tools/tht/internal/compose" "github.com/aritmolab/thothii/tools/tht/internal/config" "github.com/aritmolab/thothii/tools/tht/internal/doctor" + "github.com/aritmolab/thothii/tools/tht/internal/lifecycle" "github.com/aritmolab/thothii/tools/tht/internal/output" "github.com/aritmolab/thothii/tools/tht/internal/pi" "github.com/aritmolab/thothii/tools/tht/internal/project" @@ -42,6 +44,8 @@ Commands: start [--build] Start the installation; --build builds current-checkout images first. stop Stop the installation. update --check-only Validate the current installation without changing containers. + backup [--output PATH] [--include-secrets --yes] [--drain] + Create one transactional installation backup. sessions migrate --yes Run only the server session migrator and verify pending=[] and drifted=[]. remove Display exact stopped app container IDs without mutation. @@ -169,6 +173,8 @@ func run(ctx context.Context, args []string, stdout, stderr io.Writer) int { return commandUsageError(stderr, "update currently requires --check-only") } result, err = runner.Run(ctx, installation.ComposeArgs("config", "--quiet"), nil) + case "backup": + return backupCommand(ctx, installation, commandArgs, stdout, stderr) case "doctor": return doctorCommand(ctx, installation, runner, commandArgs, stdout, stderr) case "pi": @@ -419,6 +425,13 @@ func piCommand(ctx context.Context, installation config.Installation, runner com if len(args) == 0 { return commandUsageError(stderr, "pi requires a subcommand") } + if piMutationRequiresLifecycleLock(args) { + lock, err := lifecycle.Acquire(installation) + if err != nil { + return lifecycleFailure(stderr, err, secretValues) + } + defer func() { _ = lock.Release() }() + } controlled := compose.InstallationRunner{Installation: installation, Runner: runner} switch args[0] { case "status": @@ -790,6 +803,98 @@ func logsArgs(args []string) ([]string, error) { return nil, errors.New("logs does not accept arguments; use bounded snapshots") } +type backupExecutor func(context.Context, config.Installation, backup.CreateRequest) (backup.Result, error) + +func backupCommand(ctx context.Context, installation config.Installation, args []string, stdout, stderr io.Writer) int { + return backupCommandWith(ctx, installation, args, backup.Create, stdout, stderr) +} + +func backupCommandWith( + ctx context.Context, + installation config.Installation, + args []string, + execute backupExecutor, + stdout, stderr io.Writer, +) int { + request, err := parseBackupArgs(args) + if err != nil { + return commandUsageError(stderr, err.Error()) + } + if execute == nil { + return commandUsageError(stderr, "backup executor is unavailable") + } + result, err := execute(ctx, installation, request) + if err != nil { + message := output.Sanitize(err.Error(), nil) + fmt.Fprintf(stderr, "tht: %s\n", message) + if errors.Is(err, backup.ErrActiveSessions) || errors.Is(err, backup.ErrConfirmationRequired) || errors.Is(err, lifecycle.ErrLocked) { + return 2 + } + return 1 + } + fmt.Fprintf(stdout, "Backup created: %s\n", result.Path) + if result.Warning != "" { + fmt.Fprintln(stderr, output.Sanitize(result.Warning, nil)) + } + return 0 +} + +func parseBackupArgs(args []string) (backup.CreateRequest, error) { + request := backup.CreateRequest{} + for len(args) > 0 { + option := args[0] + args = args[1:] + switch option { + case "--output": + if request.Output != "" { + return backup.CreateRequest{}, errors.New("--output may be supplied once") + } + if len(args) == 0 || args[0] == "" { + return backup.CreateRequest{}, errors.New("--output requires a path") + } + request.Output, args = args[0], args[1:] + case "--include-secrets": + if request.IncludeSecrets { + return backup.CreateRequest{}, errors.New("--include-secrets may be supplied once") + } + request.IncludeSecrets = true + case "--yes": + if request.Confirm { + return backup.CreateRequest{}, errors.New("--yes may be supplied once") + } + request.Confirm = true + case "--drain": + if request.Drain { + return backup.CreateRequest{}, errors.New("--drain may be supplied once") + } + request.Drain = true + default: + return backup.CreateRequest{}, fmt.Errorf("unknown backup option %q", option) + } + } + if request.IncludeSecrets && !request.Confirm { + return backup.CreateRequest{}, errors.New("--include-secrets requires --yes") + } + if request.Confirm && !request.IncludeSecrets { + return backup.CreateRequest{}, errors.New("--yes is only valid with --include-secrets") + } + return request, nil +} + +func piMutationRequiresLifecycleLock(args []string) bool { + if len(args) == 0 { + return false + } + switch args[0] { + case "configure", "restart", "update", "rollback": + return true + case "maintenance": + return len(args) > 1 && args[1] == "recover" + default: + return false + } +} + func commandUsageError(stderr io.Writer, message string) int { fmt.Fprintf(stderr, "tht: %s\n", message) return 2 diff --git a/tools/tht/cmd/tht/main_test.go b/tools/tht/cmd/tht/main_test.go index 3e55ea7b..ca16fced 100644 --- a/tools/tht/cmd/tht/main_test.go +++ b/tools/tht/cmd/tht/main_test.go @@ -12,6 +12,7 @@ import ( "strings" "testing" + "github.com/aritmolab/thothii/tools/tht/internal/backup" "github.com/aritmolab/thothii/tools/tht/internal/compose" "github.com/aritmolab/thothii/tools/tht/internal/config" "github.com/aritmolab/thothii/tools/tht/internal/doctor" @@ -20,6 +21,80 @@ import ( "github.com/aritmolab/thothii/tools/tht/internal/testsupport" ) +func TestBackupCommandParsesSafeTransactionalOptions(t *testing.T) { + request, err := parseBackupArgs([]string{"--output", "backup.zip", "--include-secrets", "--yes", "--drain"}) + if err != nil { + t.Fatal(err) + } + if request.Output != "backup.zip" || !request.IncludeSecrets || !request.Confirm || !request.Drain { + t.Fatalf("backup request = %#v", request) + } + + for _, args := range [][]string{ + {"--include-secrets"}, + {"--yes"}, + {"--output"}, + {"--drain", "--drain"}, + {"--unknown"}, + } { + if _, err := parseBackupArgs(args); err == nil { + t.Errorf("parseBackupArgs(%v) succeeded", args) + } + } +} + +func TestBackupCommandDispatchesWithoutDockerAndPrintsCustodyWarning(t *testing.T) { + installation := config.Installation{Path: "/tmp/thothii-installation.yaml"} + var received backup.CreateRequest + executor := func(_ context.Context, got config.Installation, request backup.CreateRequest) (backup.Result, error) { + if got.Path != installation.Path { + t.Fatalf("installation = %#v", got) + } + received = request + return backup.Result{Path: "/tmp/backup.zip", Warning: "archive contains external secret files; protect its custody"}, nil + } + var stdout, stderr bytes.Buffer + code := backupCommandWith(context.Background(), installation, []string{"--include-secrets", "--yes"}, executor, &stdout, &stderr) + if code != 0 { + t.Fatalf("backup exit = %d, stderr = %s", code, stderr.String()) + } + if !received.IncludeSecrets || !received.Confirm || !strings.Contains(stdout.String(), "/tmp/backup.zip") || !strings.Contains(stderr.String(), "protect its custody") { + t.Fatalf("request=%#v stdout=%q stderr=%q", received, stdout.String(), stderr.String()) + } +} + +func TestBackupCommandMapsSafetyRefusalsToUsageExit(t *testing.T) { + executor := func(context.Context, config.Installation, backup.CreateRequest) (backup.Result, error) { + return backup.Result{}, backup.ErrActiveSessions + } + var stdout, stderr bytes.Buffer + if code := backupCommandWith(context.Background(), config.Installation{}, nil, executor, &stdout, &stderr); code != 2 { + t.Fatalf("backup safety refusal exit = %d, stderr = %q", code, stderr.String()) + } +} + +func TestPiMutationsUseTheSharedInstallationLifecycleLock(t *testing.T) { + for _, test := range []struct { + args []string + want bool + }{ + {args: []string{"status"}, want: false}, + {args: []string{"doctor"}, want: false}, + {args: []string{"test"}, want: false}, + {args: []string{"logs"}, want: false}, + {args: []string{"configure"}, want: true}, + {args: []string{"restart"}, want: true}, + {args: []string{"update"}, want: true}, + {args: []string{"rollback"}, want: true}, + {args: []string{"maintenance", "recover"}, want: true}, + {args: []string{"maintenance", "status"}, want: false}, + } { + if got := piMutationRequiresLifecycleLock(test.args); got != test.want { + t.Errorf("piMutationRequiresLifecycleLock(%v) = %v, want %v", test.args, got, test.want) + } + } +} + func TestInstallationRunnerMapsProfileToSessionInventoryScope(t *testing.T) { for _, test := range []struct { profile string diff --git a/tools/tht/internal/backup/create.go b/tools/tht/internal/backup/create.go new file mode 100644 index 00000000..d0b9692d --- /dev/null +++ b/tools/tht/internal/backup/create.go @@ -0,0 +1,761 @@ +// Package backup creates portable, transactional ThothII installation archives. +package backup + +import ( + "archive/zip" + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "sort" + "strings" + "time" + + "github.com/aritmolab/thothii/tools/tht/internal/compose" + "github.com/aritmolab/thothii/tools/tht/internal/config" + "github.com/aritmolab/thothii/tools/tht/internal/lifecycle" + "github.com/aritmolab/thothii/tools/tht/internal/service" +) + +var ( + // ErrActiveSessions keeps the refusal machine-readable for the host CLI. + ErrActiveSessions = errors.New("active sessions require --drain before backup") + // ErrConfirmationRequired guards backups that would contain external secret payloads. + ErrConfirmationRequired = errors.New("--include-secrets requires --yes") +) + +var requiredVolumes = []string{ + "settings", + "pi-state", + "workspace-registry", + "workspace-secrets", + "sessions", + "qdrant-data", + "embedding-models", +} + +const ( + helperImage = "busybox:1.36.1" + drainPollInterval = time.Second + maxDrainPolls = 300 +) + +// CreateRequest controls one explicit backup request. +type CreateRequest struct { + Output string + IncludeSecrets bool + Confirm bool + Drain bool +} + +// Result describes a published archive without exposing its contents. +type Result struct { + Path string + Warning string +} + +// archiveRunner separates the only streaming Docker call from regular Compose commands. Tests +// provide a deterministic in-memory implementation; production always passes argument arrays. +type archiveRunner interface { + compose.Runner + Stream(context.Context, []string, io.Reader, io.Writer) (compose.Result, error) + SessionInventoryScope() string +} + +type dependencies struct { + runner archiveRunner + now func() time.Time + homeDir func() (string, error) + revision func(context.Context, string) (string, error) + sleep func(time.Duration) + rename func(string, string) error +} + +// Create creates an archive with the real Docker command boundary. It performs no shell +// interpolation and never supplies secret contents in process arguments. +func Create(ctx context.Context, installation config.Installation, request CreateRequest) (Result, error) { + runner := hostRunner{runner: compose.NewRunner(""), binary: "docker", profile: installation.Profile} + return createWithDependencies(ctx, installation, request, dependencies{ + runner: runner, + now: time.Now, + homeDir: os.UserHomeDir, + revision: func(ctx context.Context, directory string) (string, error) { + command := exec.CommandContext(ctx, "git", "-C", directory, "rev-parse", "HEAD") + value, err := command.Output() + if err != nil { + return "", errors.New("source revision is unavailable") + } + return strings.TrimSpace(string(value)), nil + }, + sleep: time.Sleep, + rename: os.Rename, + }) +} + +func createWithDependencies(ctx context.Context, installation config.Installation, request CreateRequest, dependencies dependencies) (result Result, resultErr error) { + if dependencies.runner == nil || dependencies.now == nil || dependencies.homeDir == nil || dependencies.revision == nil || dependencies.sleep == nil || dependencies.rename == nil { + return Result{}, errors.New("backup dependencies are incomplete") + } + if request.IncludeSecrets && !request.Confirm { + return Result{}, ErrConfirmationRequired + } + lock, err := lifecycle.Acquire(installation) + if err != nil { + return Result{}, err + } + defer func() { + if releaseErr := lock.Release(); releaseErr != nil && resultErr == nil { + result = Result{} + resultErr = releaseErr + } + }() + + if err := rejectInlineSecretValues(installation.EnvFile); err != nil { + return Result{}, err + } + installationID, err := backupInstallationID(installation) + if err != nil { + return Result{}, err + } + secretPaths, err := installation.SecretFiles() + if err != nil { + return Result{}, fmt.Errorf("installation external secret references could not be read: %w", err) + } + revision, err := dependencies.revision(ctx, installation.ProjectDirectory) + if err != nil { + return Result{}, err + } + if !revisionPattern.MatchString(revision) { + return Result{}, errors.New("source revision is invalid") + } + output, err := backupOutputPath(request.Output, installationID, revision, dependencies.now().UTC(), dependencies.homeDir) + if err != nil { + return Result{}, err + } + if err := ensureNewArchivePath(output); err != nil { + return Result{}, err + } + + rendered, err := renderedConfiguration(ctx, installation, dependencies.runner) + if err != nil { + return Result{}, err + } + volumes, err := inspectRequiredVolumes(ctx, dependencies.runner, rendered) + if err != nil { + return Result{}, err + } + images, err := imageIdentities(ctx, installation, dependencies.runner, rendered) + if err != nil { + return Result{}, err + } + + wasRunning, err := installationRunning(ctx, installation, dependencies.runner) + if err != nil { + return Result{}, err + } + maintenanceActive := false + stopped := false + defer func() { + if stopped { + if startErr := composeStartAndVerify(ctx, installation, dependencies.runner); startErr != nil && resultErr == nil { + result = Result{} + resultErr = startErr + } + } + if maintenanceActive { + if deactivateErr := maintenance(ctx, installation, dependencies.runner, false); deactivateErr != nil && resultErr == nil { + result = Result{} + resultErr = deactivateErr + } + } + }() + + if wasRunning { + if err := maintenance(ctx, installation, dependencies.runner, true); err != nil { + return Result{}, err + } + maintenanceActive = true + if err := waitForNoActiveSessions(ctx, installation, dependencies.runner, request.Drain, dependencies.sleep); err != nil { + return Result{}, err + } + if err := runCompose(ctx, installation, dependencies.runner, "stop"); err != nil { + return Result{}, err + } + stopped = true + } + + manifest := Manifest{ + SchemaVersion: CurrentSchemaVersion, + InstallationID: installationID, + CreatedAt: dependencies.now().UTC(), + SourceRevision: revision, + IncludesSecrets: request.IncludeSecrets && len(secretPaths) > 0, + ComposeProject: installation.ProjectName(), + Images: images, + Volumes: volumes, + } + if err := writeArchive(ctx, output, installation, request, secretPaths, manifest, volumes, dependencies); err != nil { + return Result{}, err + } + if wasRunning { + if err := composeStartAndVerify(ctx, installation, dependencies.runner); err != nil { + return Result{}, err + } + stopped = false + if err := maintenance(ctx, installation, dependencies.runner, false); err != nil { + return Result{}, err + } + maintenanceActive = false + } + result = Result{Path: output} + if request.IncludeSecrets { + result.Warning = "The archive contains external secret files. Protect its custody and access." + } + return result, nil +} + +type hostRunner struct { + runner compose.Runner + binary string + profile string +} + +func (runner hostRunner) Run(ctx context.Context, args []string, stdin io.Reader) (compose.Result, error) { + return runner.runner.Run(ctx, args, stdin) +} + +func (runner hostRunner) Stream(ctx context.Context, args []string, stdin io.Reader, stdout io.Writer) (compose.Result, error) { + command := exec.CommandContext(ctx, runner.binary, args...) + command.Stdin = stdin + command.Stdout = stdout + var stderr bytes.Buffer + command.Stderr = &stderr + err := command.Run() + result := compose.Result{Stderr: stderr.String()} + if exitErr := new(exec.ExitError); errors.As(err, &exitErr) { + result.ExitCode = exitErr.ExitCode() + } + return result, err +} + +func (runner hostRunner) SessionInventoryScope() string { + if runner.profile == "local" { + return "mine" + } + return "all" +} + +func backupInstallationID(installation config.Installation) (string, error) { + value := filepath.Base(filepath.Dir(installation.Path)) + if value == "" || value == "." || value == ".." || strings.ContainsAny(value, `/\\`) { + return "", errors.New("installation ID is invalid") + } + return value, nil +} + +func backupOutputPath(requested, installationID, revision string, now time.Time, homeDir func() (string, error)) (string, error) { + if requested != "" { + if !filepath.IsAbs(requested) { + absolute, err := filepath.Abs(requested) + if err != nil { + return "", errors.New("backup output path is unavailable") + } + requested = absolute + } + return filepath.Clean(requested), nil + } + home, err := homeDir() + if err != nil || home == "" { + return "", errors.New("home directory is unavailable for the default backup path") + } + name := fmt.Sprintf("thothii-%s-%s-%s.zip", installationID, now.UTC().Format("20060102T150405Z"), revision) + return filepath.Join(home, ".thothii", "backups", installationID, name), nil +} + +func ensureNewArchivePath(output string) error { + if output == "" || !filepath.IsAbs(output) { + return errors.New("backup output path must be absolute") + } + if err := os.MkdirAll(filepath.Dir(output), 0o700); err != nil { + return fmt.Errorf("create backup directory: %w", err) + } + if info, err := os.Lstat(output); err == nil || !errors.Is(err, os.ErrNotExist) { + if err == nil && info.Mode().IsRegular() { + return errors.New("backup output already exists") + } + return errors.New("backup output path is unavailable") + } + return nil +} + +type renderedCompose struct { + Volumes map[string]struct { + Name string `json:"name"` + } `json:"volumes"` + Services map[string]struct { + Image string `json:"image"` + } `json:"services"` +} + +func renderedConfiguration(ctx context.Context, installation config.Installation, runner archiveRunner) (renderedCompose, error) { + result, err := runner.Run(ctx, installation.ComposeArgs("config", "--format", "json"), nil) + if err != nil { + return renderedCompose{}, dockerError("render Compose configuration", result, err) + } + var rendered renderedCompose + if json.Unmarshal([]byte(result.Stdout), &rendered) != nil { + return renderedCompose{}, errors.New("Docker Compose returned invalid backup configuration") + } + for _, logical := range requiredVolumes { + if rendered.Volumes[logical].Name == "" { + return renderedCompose{}, fmt.Errorf("required backup volume %q is not configured", logical) + } + } + return rendered, nil +} + +func inspectRequiredVolumes(ctx context.Context, runner archiveRunner, rendered renderedCompose) ([]VolumeMetadata, error) { + names := make([]string, 0, len(requiredVolumes)) + for _, logical := range requiredVolumes { + names = append(names, rendered.Volumes[logical].Name) + } + result, err := runner.Run(ctx, append([]string{"volume", "inspect"}, names...), nil) + if err != nil { + return nil, dockerError("inspect backup volumes", result, err) + } + var inspected []struct { + Name string `json:"Name"` + Driver string `json:"Driver"` + Labels map[string]string `json:"Labels"` + } + if json.Unmarshal([]byte(result.Stdout), &inspected) != nil { + return nil, errors.New("Docker returned invalid volume metadata") + } + byName := make(map[string]struct { + Name string + Driver string + Labels map[string]string + }, len(inspected)) + for _, item := range inspected { + byName[item.Name] = struct { + Name string + Driver string + Labels map[string]string + }{item.Name, item.Driver, item.Labels} + } + volumes := make([]VolumeMetadata, 0, len(requiredVolumes)) + for _, logical := range requiredVolumes { + name := rendered.Volumes[logical].Name + item, ok := byName[name] + if !ok || item.Driver == "" { + return nil, fmt.Errorf("required backup volume %q is unavailable", logical) + } + volumes = append(volumes, VolumeMetadata{LogicalName: logical, Name: name, Driver: item.Driver, Labels: item.Labels}) + } + return volumes, nil +} + +func imageIdentities(ctx context.Context, installation config.Installation, runner archiveRunner, rendered renderedCompose) ([]ImageIdentity, error) { + result, err := runner.Run(ctx, installation.ComposeArgs("images", "--format", "json"), nil) + if err != nil { + return nil, dockerError("inspect image identities", result, err) + } + ids := map[string]string{} + decoder := json.NewDecoder(strings.NewReader(result.Stdout)) + for { + var item struct { + Service string `json:"Service"` + ID string `json:"ID"` + } + err := decoder.Decode(&item) + if errors.Is(err, io.EOF) { + break + } + if err != nil || item.Service == "" { + return nil, errors.New("Docker Compose returned invalid image identities") + } + ids[item.Service] = item.ID + } + services := make([]string, 0, len(rendered.Services)) + for name, definition := range rendered.Services { + if definition.Image != "" { + services = append(services, name) + } + } + sort.Strings(services) + images := make([]ImageIdentity, 0, len(services)) + for _, name := range services { + images = append(images, ImageIdentity{Service: name, Reference: rendered.Services[name].Image, ID: ids[name]}) + } + return images, nil +} + +func installationRunning(ctx context.Context, installation config.Installation, runner archiveRunner) (bool, error) { + result, err := runner.Run(ctx, installation.ComposeArgs("ps", "--status", "running", "-q"), nil) + if err != nil { + return false, dockerError("inspect running services", result, err) + } + return strings.TrimSpace(result.Stdout) != "", nil +} + +func maintenance(ctx context.Context, installation config.Installation, runner archiveRunner, activate bool) error { + action := "deactivate" + if activate { + action = "activate" + } + result, err := runner.Run(ctx, installation.ComposeArgs( + "exec", "-T", "core", "curl", "-fsS", "--max-time", "5", "-X", "POST", + "http://127.0.0.1:8787/internal/maintenance/"+action, + ), nil) + if err != nil { + return dockerError("change maintenance admissions", result, err) + } + return nil +} + +func waitForNoActiveSessions(ctx context.Context, installation config.Installation, runner archiveRunner, drain bool, sleep func(time.Duration)) error { + for attempt := 0; attempt < maxDrainPolls; attempt++ { + result, err := runner.Run(ctx, installation.ComposeArgs( + "exec", "-T", "core", "curl", "-fsS", "--max-time", "5", + "http://127.0.0.1:8787/sessions?scope="+runner.SessionInventoryScope(), + ), nil) + if err != nil { + return dockerError("inspect active sessions", result, err) + } + active, err := activeSessions(result.Stdout) + if err != nil { + return err + } + if !active { + return nil + } + if !drain { + return ErrActiveSessions + } + select { + case <-ctx.Done(): + return ctx.Err() + default: + sleep(drainPollInterval) + } + } + return errors.New("active sessions did not drain before the backup deadline") +} + +func activeSessions(value string) (bool, error) { + var sessions []struct { + Status string `json:"status"` + Archived bool `json:"archived"` + } + if json.Unmarshal([]byte(value), &sessions) != nil { + return false, errors.New("session inventory is invalid") + } + for _, session := range sessions { + if !session.Archived && (session.Status == "" || session.Status == "running" || session.Status == "active") { + return true, nil + } + } + return false, nil +} + +func composeStartAndVerify(ctx context.Context, installation config.Installation, runner archiveRunner) error { + if err := runCompose(ctx, installation, runner, "start"); err != nil { + return err + } + return service.WaitForHealthy(ctx, installation, runner) +} + +func runCompose(ctx context.Context, installation config.Installation, runner archiveRunner, command ...string) error { + result, err := runner.Run(ctx, installation.ComposeArgs(command...), nil) + if err != nil { + return dockerError("run Docker Compose", result, err) + } + return nil +} + +func writeArchive(ctx context.Context, output string, installation config.Installation, request CreateRequest, secretPaths []string, manifest Manifest, volumes []VolumeMetadata, dependencies dependencies) (resultErr error) { + directory := filepath.Dir(output) + temporary, err := os.CreateTemp(directory, ".tht-backup-*.tmp") + if err != nil { + return fmt.Errorf("create temporary backup archive: %w", err) + } + temporaryPath := temporary.Name() + published := false + closedFile := false + defer func() { + var closeErr error + if !closedFile { + closeErr = temporary.Close() + } + if closeErr != nil && resultErr == nil { + resultErr = closeErr + } + if !published { + _ = os.Remove(temporaryPath) + _ = os.Remove(output) + } + }() + if err := temporary.Chmod(0o600); err != nil { + return fmt.Errorf("protect temporary backup archive: %w", err) + } + writer := zip.NewWriter(temporary) + closedWriter := false + defer func() { + if !closedWriter { + _ = writer.Close() + } + }() + + add := func(name string, mode os.FileMode, input io.Reader) (Entry, error) { + header := &zip.FileHeader{Name: filepath.ToSlash(name), Method: zip.Deflate} + header.SetMode(mode) + created, err := writer.CreateHeader(header) + if err != nil { + return Entry{}, err + } + hash := sha256.New() + size, err := io.Copy(io.MultiWriter(created, hash), input) + if err != nil { + return Entry{}, err + } + return Entry{Path: filepath.ToSlash(name), SHA256: "sha256:" + hex.EncodeToString(hash.Sum(nil)), Size: size, Archived: true, Mode: uint32(mode.Perm())}, nil + } + addFile := func(name, owner string, sensitive bool, source string) error { + file, err := os.Open(source) + if err != nil { + return fmt.Errorf("read backup input: %w", err) + } + defer file.Close() + info, err := file.Stat() + if err != nil || !info.Mode().IsRegular() { + return errors.New("backup input is not a regular file") + } + entry, err := add(name, info.Mode(), file) + if err != nil { + return err + } + entry.Kind, entry.Owner, entry.Sensitive = EntryFile, owner, sensitive + manifest.Entries = append(manifest.Entries, entry) + return nil + } + + for _, input := range configurationInputs(installation) { + if input.optional { + if _, err := os.Stat(input.source); errors.Is(err, os.ErrNotExist) { + continue + } + } + if err := addFile(input.name, input.owner, false, input.source); err != nil { + return err + } + } + for index, source := range secretPaths { + if request.IncludeSecrets { + if err := addFile(fmt.Sprintf("external-secrets/%03d", index), "external-secret", true, source); err != nil { + return err + } + entry := &manifest.Entries[len(manifest.Entries)-1] + entry.Kind, entry.SourcePath, entry.Sensitive = EntryExternalSecret, source, true + continue + } + file, err := os.Open(source) + if err != nil { + return errors.New("external secret file is unavailable") + } + info, statErr := file.Stat() + if statErr != nil || !info.Mode().IsRegular() { + _ = file.Close() + return errors.New("external secret file is unavailable") + } + hash := sha256.New() + size, copyErr := io.Copy(hash, file) + closeErr := file.Close() + if copyErr != nil || closeErr != nil { + return errors.New("external secret file is unavailable") + } + manifest.Entries = append(manifest.Entries, Entry{ + Path: fmt.Sprintf("external-secrets/%03d", index), Kind: EntrySecretReference, Owner: "external-secret", + SourcePath: source, SHA256: "sha256:" + hex.EncodeToString(hash.Sum(nil)), Size: size, Sensitive: true, + }) + } + for _, volume := range volumes { + headerName := "volumes/" + volume.LogicalName + ".tar" + header := &zip.FileHeader{Name: headerName, Method: zip.Deflate} + header.SetMode(0o600) + archiveWriter, err := writer.CreateHeader(header) + if err != nil { + return fmt.Errorf("create volume archive entry %s: %w", volume.LogicalName, err) + } + hash := sha256.New() + sizeWriter := &countingWriter{writer: io.MultiWriter(archiveWriter, hash)} + streamResult, streamErr := dependencies.runner.Stream(ctx, volumeArchiveCommand(volume.Name), nil, sizeWriter) + if streamErr != nil || streamResult.ExitCode != 0 { + if streamErr == nil { + streamErr = errors.New("Docker volume helper returned a nonzero exit status") + } + return fmt.Errorf("archive volume %s: %w", volume.LogicalName, dockerError("stream volume", streamResult, streamErr)) + } + entry := Entry{ + Path: headerName, Kind: EntryVolume, Owner: "volume:" + volume.LogicalName, LogicalName: volume.LogicalName, + SHA256: "sha256:" + hex.EncodeToString(hash.Sum(nil)), Size: sizeWriter.size, Mode: 0o600, Archived: true, + Sensitive: volume.LogicalName == "workspace-secrets", + } + manifest.Entries = append(manifest.Entries, entry) + } + if installation.Profile == "server" { + if err := archivePreservationRoots(installation, secretPaths, addFile); err != nil { + return err + } + } + manifestJSON, err := manifest.JSON() + if err != nil { + return err + } + if _, err := add(ManifestPath, 0o600, bytes.NewReader(manifestJSON)); err != nil { + return err + } + if err := writer.Close(); err != nil { + return fmt.Errorf("close backup archive: %w", err) + } + closedWriter = true + if err := temporary.Sync(); err != nil { + return fmt.Errorf("fsync backup archive: %w", err) + } + if err := temporary.Close(); err != nil { + return fmt.Errorf("close backup archive: %w", err) + } + closedFile = true + if err := dependencies.rename(temporaryPath, output); err != nil { + return fmt.Errorf("publish backup archive: %w", err) + } + published = true + return nil +} + +type configurationInput struct { + name, owner, source string + optional bool +} + +func configurationInputs(installation config.Installation) []configurationInput { + inputs := []configurationInput{ + {"configuration/installation/thothii-installation.yaml", "installation", installation.Path, false}, + {"configuration/environment/operator.env", "installation", installation.EnvFile, false}, + {"configuration/pi/models.json", "pi", filepath.Join(installation.ProjectDirectory, "deploy", "pi", "models.json"), false}, + {"configuration/pi/settings.json", "pi", filepath.Join(installation.ProjectDirectory, "deploy", "pi", "settings.json"), false}, + {"configuration/generated/current-image.yaml", "installation", installation.CurrentImageOverridePath(), true}, + } + for index, source := range installation.Overrides { + inputs = append(inputs, configurationInput{fmt.Sprintf("configuration/overrides/%02d-%s", index, filepath.Base(source)), "installation", source, false}) + } + return inputs +} + +func volumeArchiveCommand(volume string) []string { + return []string{"run", "--rm", "--network", "none", "--mount", "type=volume,src=" + volume + ",dst=/source,readonly", helperImage, "tar", "--numeric-owner", "-C", "/source", "-cf", "-", "."} +} + +type countingWriter struct { + writer io.Writer + size int64 +} + +func (writer *countingWriter) Write(value []byte) (int, error) { + written, err := writer.writer.Write(value) + writer.size += int64(written) + return written, err +} + +func archivePreservationRoots(installation config.Installation, secretPaths []string, addFile func(string, string, bool, string) error) error { + backupRoot, err := installation.EnvironmentValue("THT_BACKUP_ROOT") + if err != nil { + return errors.New("server preservation roots are unavailable") + } + secretSet := make(map[string]bool, len(secretPaths)) + for _, secret := range secretPaths { + secretSet[secret] = true + } + for index, variable := range []string{"THT_DATA_ROOT", "THT_PI_STATE_ROOT", "THT_WORKSPACE_REGISTRY_ROOT"} { + root, err := installation.EnvironmentValue(variable) + if err != nil || root == "" { + return errors.New("server preservation roots are unavailable") + } + info, err := os.Stat(root) + if err != nil || !info.IsDir() { + return errors.New("server preservation root is unavailable") + } + err = filepath.WalkDir(root, func(path string, entry os.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if path == backupRoot { + if entry.IsDir() { + return filepath.SkipDir + } + return nil + } + if entry.Type()&os.ModeSymlink != 0 { + return errors.New("server preservation root contains a symlink") + } + if entry.IsDir() || secretSet[path] { + return nil + } + if !entry.Type().IsRegular() { + return errors.New("server preservation root contains an unsupported entry") + } + relative, err := filepath.Rel(root, path) + if err != nil { + return err + } + name := fmt.Sprintf("preservation/%02d-%s/%s", index, filepath.Base(root), filepath.ToSlash(relative)) + return addFile(name, "preservation-root:"+variable, false, path) + }) + if err != nil { + return err + } + } + return nil +} + +func rejectInlineSecretValues(environmentPath string) error { + contents, err := os.ReadFile(environmentPath) + if err != nil { + return errors.New("installation environment could not be read") + } + for _, line := range strings.Split(string(contents), "\n") { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + line = strings.TrimPrefix(line, "export ") + key, value, found := strings.Cut(line, "=") + if !found || strings.TrimSpace(value) == "" { + continue + } + key = strings.ToUpper(strings.TrimSpace(key)) + if strings.HasSuffix(key, "_FILE") || strings.HasSuffix(key, "_SOURCE") { + continue + } + if strings.Contains(key, "SECRET") || strings.Contains(key, "PASSWORD") || strings.Contains(key, "TOKEN") || strings.Contains(key, "API_KEY") || strings.Contains(key, "CREDENTIAL") || strings.Contains(key, "AUTHORIZATION") || strings.HasSuffix(key, "_AUTH") { + return errors.New("installation environment contains an inline secret value") + } + } + return nil +} + +func dockerError(action string, result compose.Result, err error) error { + if result.ExitCode != 0 { + return fmt.Errorf("%s: Docker exited with status %d", action, result.ExitCode) + } + if err != nil { + return fmt.Errorf("%s: %w", action, err) + } + return fmt.Errorf("%s failed", action) +} diff --git a/tools/tht/internal/backup/create_test.go b/tools/tht/internal/backup/create_test.go new file mode 100644 index 00000000..08528d4e --- /dev/null +++ b/tools/tht/internal/backup/create_test.go @@ -0,0 +1,638 @@ +package backup + +import ( + "archive/tar" + "archive/zip" + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strings" + "testing" + "time" + + "github.com/aritmolab/thothii/tools/tht/internal/compose" + "github.com/aritmolab/thothii/tools/tht/internal/config" + "github.com/aritmolab/thothii/tools/tht/internal/lifecycle" +) + +var requiredTestVolumes = []string{"settings", "pi-state", "workspace-registry", "workspace-secrets", "sessions", "qdrant-data", "embedding-models"} + +func TestCreateWritesManifestLastWithConfigurationMetadataAndSevenVolumes(t *testing.T) { + fixture := newBackupFixture(t, "local") + output := filepath.Join(t.TempDir(), "custom.zip") + runner := newBackupRunner(fixture.installation, false) + + result, err := createWithDependencies(context.Background(), fixture.installation, CreateRequest{Output: output}, testDependencies(t, runner)) + if err != nil { + t.Fatal(err) + } + if result.Path != output { + t.Fatalf("Result.Path = %q, want %q", result.Path, output) + } + archive := readFixtureArchive(t, output) + if got := archive.order[len(archive.order)-1]; got != ManifestPath { + t.Fatalf("last archive entry = %q, want %q", got, ManifestPath) + } + if archive.manifest.InstallationID != fixture.installationID || archive.manifest.SourceRevision != testRevision || archive.manifest.ComposeProject != fixture.installation.ProjectName() { + t.Fatalf("manifest identity = %#v", archive.manifest) + } + if len(archive.manifest.Volumes) != len(requiredTestVolumes) { + t.Fatalf("manifest volumes = %d, want %d", len(archive.manifest.Volumes), len(requiredTestVolumes)) + } + for _, logical := range requiredTestVolumes { + path := "volumes/" + logical + ".tar" + if _, exists := archive.files[path]; !exists { + t.Errorf("archive is missing %s", path) + } + } + for _, path := range []string{ + "configuration/installation/thothii-installation.yaml", + "configuration/environment/operator.env", + "configuration/pi/models.json", + "configuration/pi/settings.json", + "configuration/generated/current-image.yaml", + } { + if _, exists := archive.files[path]; !exists { + t.Errorf("archive is missing %s", path) + } + } + if len(archive.manifest.Images) < 2 { + t.Fatalf("image identities = %#v", archive.manifest.Images) + } + if runner.streamWhileRunning { + t.Fatal("a volume was streamed before the installation was stopped") + } +} + +func TestCreateRestartsAndVerifiesAnInstallationThatWasRunning(t *testing.T) { + fixture := newBackupFixture(t, "local") + runner := newBackupRunner(fixture.installation, true) + output := filepath.Join(t.TempDir(), "running.zip") + + if _, err := createWithDependencies(context.Background(), fixture.installation, CreateRequest{Output: output}, testDependencies(t, runner)); err != nil { + t.Fatal(err) + } + if !runner.running || !runner.coreRunning || runner.maintenance { + t.Fatalf("running state was not restored: running=%v core=%v maintenance=%v", runner.running, runner.coreRunning, runner.maintenance) + } + if runner.stopCount != 1 || runner.startCount != 1 || runner.healthChecks == 0 { + t.Fatalf("lifecycle counts: stop=%d start=%d health=%d", runner.stopCount, runner.startCount, runner.healthChecks) + } +} + +func TestCreateRefusesActiveSessionsWithoutDrainAndRestoresAdmissions(t *testing.T) { + fixture := newBackupFixture(t, "local") + runner := newBackupRunner(fixture.installation, true) + runner.sessionResponses = []string{activeSessionPayload()} + output := filepath.Join(t.TempDir(), "refused.zip") + + _, err := createWithDependencies(context.Background(), fixture.installation, CreateRequest{Output: output}, testDependencies(t, runner)) + if !errors.Is(err, ErrActiveSessions) { + t.Fatalf("Create() error = %v, want ErrActiveSessions", err) + } + if runner.stopCount != 0 || !runner.running || runner.maintenance { + t.Fatalf("refusal changed lifecycle state: stop=%d running=%v maintenance=%v", runner.stopCount, runner.running, runner.maintenance) + } + if _, statErr := os.Stat(output); !errors.Is(statErr, os.ErrNotExist) { + t.Fatalf("unsafe backup was published: %v", statErr) + } +} + +func TestCreateDrainsActiveSessionsBeforeStoppedSnapshot(t *testing.T) { + fixture := newBackupFixture(t, "local") + runner := newBackupRunner(fixture.installation, true) + runner.sessionResponses = []string{activeSessionPayload(), activeSessionPayload(), `[]`} + dependencies := testDependencies(t, runner) + sleeps := 0 + dependencies.sleep = func(time.Duration) { sleeps++ } + + if _, err := createWithDependencies(context.Background(), fixture.installation, CreateRequest{Output: filepath.Join(t.TempDir(), "drained.zip"), Drain: true}, dependencies); err != nil { + t.Fatal(err) + } + if sleeps == 0 || runner.streamWhileRunning { + t.Fatalf("drain did not wait for a stopped snapshot: sleeps=%d streamedWhileRunning=%v", sleeps, runner.streamWhileRunning) + } +} + +func TestCreateDefaultPathUsesHomeInstallationIDUTCAndSourceRevision(t *testing.T) { + fixture := newBackupFixture(t, "local") + runner := newBackupRunner(fixture.installation, false) + dependencies := testDependencies(t, runner) + home := t.TempDir() + dependencies.homeDir = func() (string, error) { return home, nil } + + result, err := createWithDependencies(context.Background(), fixture.installation, CreateRequest{}, dependencies) + if err != nil { + t.Fatal(err) + } + wantDirectory := filepath.Join(home, ".thothii", "backups", fixture.installationID) + if filepath.Dir(result.Path) != wantDirectory { + t.Fatalf("default directory = %q, want %q", filepath.Dir(result.Path), wantDirectory) + } + name := filepath.Base(result.Path) + for _, fragment := range []string{"20260816T081112Z", testRevision} { + if !strings.Contains(name, fragment) { + t.Errorf("default archive name %q does not contain %q", name, fragment) + } + } +} + +func TestCreateCleansIncompleteArchiveAndRestoresRunningStateAfterStreamFailure(t *testing.T) { + fixture := newBackupFixture(t, "local") + runner := newBackupRunner(fixture.installation, true) + runner.failStreamAt = 3 + directory := t.TempDir() + output := filepath.Join(directory, "failed.zip") + + if _, err := createWithDependencies(context.Background(), fixture.installation, CreateRequest{Output: output}, testDependencies(t, runner)); err == nil { + t.Fatal("Create() succeeded after a volume stream failure") + } + if !runner.running || !runner.coreRunning || runner.maintenance { + t.Fatalf("running state was not recovered: running=%v core=%v maintenance=%v", runner.running, runner.coreRunning, runner.maintenance) + } + assertNoBackupArtifacts(t, directory) +} + +func TestCreateCleansIncompleteArchiveWhenVolumeHelperReportsANonzeroExit(t *testing.T) { + fixture := newBackupFixture(t, "local") + runner := newBackupRunner(fixture.installation, false) + runner.exitOnlyAt = 3 + directory := t.TempDir() + + if _, err := createWithDependencies(context.Background(), fixture.installation, CreateRequest{Output: filepath.Join(directory, "failed.zip")}, testDependencies(t, runner)); err == nil { + t.Fatal("Create() succeeded after a nonzero volume-helper exit") + } + assertNoBackupArtifacts(t, directory) +} + +func TestCreateCleansIncompleteArchiveAfterAtomicPublishFailure(t *testing.T) { + fixture := newBackupFixture(t, "local") + runner := newBackupRunner(fixture.installation, false) + directory := t.TempDir() + dependencies := testDependencies(t, runner) + dependencies.rename = func(string, string) error { return errors.New("publish failed") } + + if _, err := createWithDependencies(context.Background(), fixture.installation, CreateRequest{Output: filepath.Join(directory, "failed.zip")}, dependencies); err == nil { + t.Fatal("Create() succeeded after publish failure") + } + assertNoBackupArtifacts(t, directory) +} + +func TestCreateExcludesExternalSecretPayloadsByDefaultButRecordsDigests(t *testing.T) { + fixture := newBackupFixture(t, "local") + runner := newBackupRunner(fixture.installation, false) + output := filepath.Join(t.TempDir(), "no-external-secrets.zip") + + if _, err := createWithDependencies(context.Background(), fixture.installation, CreateRequest{Output: output}, testDependencies(t, runner)); err != nil { + t.Fatal(err) + } + archive := readFixtureArchive(t, output) + all := bytes.Join(mapValues(archive.files), nil) + if bytes.Contains(all, []byte(fixture.secretValue)) { + t.Fatal("default archive contains an external secret value") + } + secretReferences := 0 + workspaceSecrets := false + for _, entry := range archive.manifest.Entries { + if entry.Kind == EntrySecretReference { + secretReferences++ + if entry.Archived || entry.SourcePath == "" || entry.SHA256 != DigestBytes([]byte(fixture.secretValue)) { + t.Fatalf("secret reference = %#v", entry) + } + } + if entry.Path == "volumes/workspace-secrets.tar" { + workspaceSecrets = entry.Archived + } + } + if secretReferences != 1 || !workspaceSecrets || archive.manifest.IncludesSecrets { + t.Fatalf("secret behavior: references=%d workspace=%v includes=%v", secretReferences, workspaceSecrets, archive.manifest.IncludesSecrets) + } + if strings.Contains(strings.Join(runner.calls, "\n"), fixture.secretValue) { + t.Fatal("external secret value appeared in a process argument") + } +} + +func TestCreateRequiresConfirmationToIncludeSecretsAndUsesOwnerOnlyMode(t *testing.T) { + fixture := newBackupFixture(t, "local") + output := filepath.Join(t.TempDir(), "with-secrets.zip") + request := CreateRequest{Output: output, IncludeSecrets: true} + if _, err := createWithDependencies(context.Background(), fixture.installation, request, testDependencies(t, newBackupRunner(fixture.installation, false))); !errors.Is(err, ErrConfirmationRequired) { + t.Fatalf("Create() error = %v, want ErrConfirmationRequired", err) + } + + request.Confirm = true + result, err := createWithDependencies(context.Background(), fixture.installation, request, testDependencies(t, newBackupRunner(fixture.installation, false))) + if err != nil { + t.Fatal(err) + } + if result.Warning == "" || strings.Contains(result.Warning, fixture.secretValue) { + t.Fatalf("custody warning = %q", result.Warning) + } + info, err := os.Stat(output) + if err != nil { + t.Fatal(err) + } + if got := info.Mode().Perm(); got != 0o600 { + t.Fatalf("archive mode = %#o, want 0600", got) + } + archive := readFixtureArchive(t, output) + if !archive.manifest.IncludesSecrets || !bytes.Contains(bytes.Join(mapValues(archive.files), nil), []byte(fixture.secretValue)) { + t.Fatal("confirmed archive does not include the external secret payload") + } +} + +func TestCreateRejectsInlineSecretValuesBeforeWritingAnArchive(t *testing.T) { + fixture := newBackupFixture(t, "local") + fixture.environment = append(fixture.environment, "THT_LLM_API_KEY=must-never-enter-an-archive") + fixture.writeEnvironment(t) + directory := t.TempDir() + runner := newBackupRunner(fixture.installation, false) + + _, err := createWithDependencies(context.Background(), fixture.installation, CreateRequest{Output: filepath.Join(directory, "unsafe.zip")}, testDependencies(t, runner)) + if err == nil || !strings.Contains(err.Error(), "inline secret") || strings.Contains(err.Error(), "must-never") { + t.Fatalf("Create() error = %v, want value-free inline-secret refusal", err) + } + if len(runner.calls) != 0 { + t.Fatalf("Docker was called before unsafe environment refusal: %v", runner.calls) + } + assertNoBackupArtifacts(t, directory) +} + +func TestCreateRejectsInlineAuthorizationValuesBeforeWritingAnArchive(t *testing.T) { + fixture := newBackupFixture(t, "local") + fixture.environment = append(fixture.environment, "DWH_AUTHORIZATION=must-never-enter-an-archive") + fixture.writeEnvironment(t) + directory := t.TempDir() + runner := newBackupRunner(fixture.installation, false) + + _, err := createWithDependencies(context.Background(), fixture.installation, CreateRequest{Output: filepath.Join(directory, "unsafe.zip")}, testDependencies(t, runner)) + if err == nil || !strings.Contains(err.Error(), "inline secret") || strings.Contains(err.Error(), "must-never") { + t.Fatalf("Create() error = %v, want value-free inline-secret refusal", err) + } + if len(runner.calls) != 0 { + t.Fatalf("Docker was called before unsafe environment refusal: %v", runner.calls) + } + assertNoBackupArtifacts(t, directory) +} + +func TestCreateIncludesServerPreservationRootsWithoutRecursingIntoBackupRoot(t *testing.T) { + fixture := newBackupFixture(t, "server") + for _, item := range []struct{ variable, name, value string }{ + {"THT_DATA_ROOT", "data", "session-state"}, + {"THT_PI_STATE_ROOT", "pi-state", "pi-state"}, + {"THT_WORKSPACE_REGISTRY_ROOT", "registry", "workspace-registry"}, + } { + root := filepath.Join(fixture.root, item.name) + if err := os.MkdirAll(root, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, "payload"), []byte(item.value), 0o600); err != nil { + t.Fatal(err) + } + fixture.environment = append(fixture.environment, item.variable+"="+root) + } + backupRoot := filepath.Join(fixture.root, "existing-backups") + if err := os.MkdirAll(backupRoot, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(backupRoot, "old-secret-backup"), []byte("must-not-be-recursed"), 0o600); err != nil { + t.Fatal(err) + } + fixture.environment = append(fixture.environment, "THT_BACKUP_ROOT="+backupRoot) + fixture.writeEnvironment(t) + output := filepath.Join(t.TempDir(), "server.zip") + + if _, err := createWithDependencies(context.Background(), fixture.installation, CreateRequest{Output: output}, testDependencies(t, newBackupRunner(fixture.installation, false))); err != nil { + t.Fatal(err) + } + archive := readFixtureArchive(t, output) + all := bytes.Join(mapValues(archive.files), nil) + for _, value := range []string{"session-state", "pi-state", "workspace-registry"} { + if !bytes.Contains(all, []byte(value)) { + t.Errorf("server preservation payload %q is missing", value) + } + } + if bytes.Contains(all, []byte("must-not-be-recursed")) { + t.Fatal("backup destination root was recursively included") + } +} + +func TestCreateHonorsTheSharedInstallationLifecycleLock(t *testing.T) { + fixture := newBackupFixture(t, "local") + lock, err := lifecycle.Acquire(fixture.installation) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = lock.Release() }) + runner := newBackupRunner(fixture.installation, false) + + _, err = createWithDependencies(context.Background(), fixture.installation, CreateRequest{Output: filepath.Join(t.TempDir(), "locked.zip")}, testDependencies(t, runner)) + if !errors.Is(err, lifecycle.ErrLocked) { + t.Fatalf("Create() error = %v, want lifecycle.ErrLocked", err) + } + if len(runner.calls) != 0 { + t.Fatalf("Docker runner was called while lock was held: %v", runner.calls) + } +} + +type backupFixture struct { + root string + installationID string + installation config.Installation + environment []string + secretValue string +} + +func newBackupFixture(t *testing.T, profile string) *backupFixture { + t.Helper() + temporaryRoot, err := filepath.EvalSymlinks(os.TempDir()) + if err != nil { + t.Fatal(err) + } + root, err := os.MkdirTemp(temporaryRoot, "tht-backup-") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.RemoveAll(root) }) + for _, directory := range []string{filepath.Join(root, "deploy", "pi"), filepath.Join(root, "deploy", "fixture")} { + if err := os.MkdirAll(directory, 0o700); err != nil { + t.Fatal(err) + } + } + write := func(path, contents string) { + if err := os.WriteFile(path, []byte(contents), 0o600); err != nil { + t.Fatal(err) + } + } + write(filepath.Join(root, "compose.yaml"), "services: {}\n") + write(filepath.Join(root, "deploy", "compose."+profile+".yaml"), "services: {}\n") + write(filepath.Join(root, "deploy", "pi", "models.json"), `{"providers":{}}`) + write(filepath.Join(root, "deploy", "pi", "settings.json"), `{"enabledModels":[]}`) + override := filepath.Join(root, "deploy", "fixture", "extra.yaml") + write(override, "services: {}\n") + installationID := profile + "-fixture" + descriptorDirectory := filepath.Join(root, "deploy", installationID) + if err := os.MkdirAll(descriptorDirectory, 0o700); err != nil { + t.Fatal(err) + } + descriptor := filepath.Join(descriptorDirectory, "thothii-installation.yaml") + environment := filepath.Join(descriptorDirectory, "operator.env") + write(descriptor, "profile: "+profile+"\nprojectDirectory: "+root+"\nenvFile: "+environment+"\n") + secretValue := "external-secret-value-for-backup-test" + secretDirectory, err := os.MkdirTemp(temporaryRoot, "tht-backup-secret-") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.RemoveAll(secretDirectory) }) + secretPath := filepath.Join(secretDirectory, "external-secret") + write(secretPath, secretValue) + fixture := &backupFixture{ + root: root, + installationID: installationID, + secretValue: secretValue, + environment: []string{ + "THT_WORKSPACE_INSTALLATION_ID=" + installationID, + "THT_SECRETS_FILE=" + secretPath, + "THT_WORKSPACE_GIT_REMOTE=https://git.example.invalid/workspaces.git", + "THT_WORKSPACE_GIT_BRANCH=main", + }, + installation: config.Installation{ + Path: descriptor, Profile: profile, ProjectDirectory: root, EnvFile: environment, + Overrides: []string{override}, + }, + } + fixture.writeEnvironment(t) + currentImage := fixture.installation.CurrentImageOverridePath() + if err := os.MkdirAll(filepath.Dir(currentImage), 0o700); err != nil { + t.Fatal(err) + } + write(currentImage, "services:\n core:\n image: core:current\n") + return fixture +} + +func (fixture *backupFixture) writeEnvironment(t *testing.T) { + t.Helper() + if err := os.WriteFile(fixture.installation.EnvFile, []byte(strings.Join(fixture.environment, "\n")+"\n"), 0o600); err != nil { + t.Fatal(err) + } +} + +type fakeBackupRunner struct { + installation config.Installation + running bool + coreRunning bool + maintenance bool + sessionResponses []string + calls []string + streams int + failStreamAt int + exitOnlyAt int + streamWhileRunning bool + stopCount int + startCount int + healthChecks int +} + +func newBackupRunner(installation config.Installation, running bool) *fakeBackupRunner { + return &fakeBackupRunner{installation: installation, running: running, coreRunning: running} +} + +func (runner *fakeBackupRunner) Run(_ context.Context, args []string, _ io.Reader) (compose.Result, error) { + command := strings.Join(args, " ") + runner.calls = append(runner.calls, command) + switch { + case strings.Contains(command, " config --format json"): + volumes := map[string]map[string]string{} + for _, logical := range requiredTestVolumes { + volumes[logical] = map[string]string{"name": runner.installation.ProjectName() + "_" + logical} + } + payload := map[string]any{ + "volumes": volumes, + "services": map[string]any{ + "core": map[string]any{"image": "thothii-core:test"}, + "frontend": map[string]any{"image": "thothii-frontend:test"}, + }, + } + encoded, _ := json.Marshal(payload) + return compose.Result{Stdout: string(encoded)}, nil + case strings.HasPrefix(command, "volume inspect "): + items := make([]map[string]any, 0, len(requiredTestVolumes)) + for _, logical := range requiredTestVolumes { + items = append(items, map[string]any{ + "Name": runner.installation.ProjectName() + "_" + logical, + "Driver": "local", + "Labels": map[string]string{"com.docker.compose.project": runner.installation.ProjectName(), "com.docker.compose.volume": logical}, + }) + } + encoded, _ := json.Marshal(items) + return compose.Result{Stdout: string(encoded)}, nil + case strings.Contains(command, " images --format json"): + return compose.Result{Stdout: "{\"Service\":\"core\",\"Repository\":\"thothii-core\",\"Tag\":\"test\",\"ID\":\"sha256:core\"}\n{\"Service\":\"frontend\",\"Repository\":\"thothii-frontend\",\"Tag\":\"test\",\"ID\":\"sha256:frontend\"}\n"}, nil + case strings.Contains(command, " ps --status running -q core"): + if runner.coreRunning { + return compose.Result{Stdout: "core-id\n"}, nil + } + return compose.Result{}, nil + case strings.Contains(command, " ps --status running -q"): + if runner.running { + return compose.Result{Stdout: "container-id\n"}, nil + } + return compose.Result{}, nil + case strings.Contains(command, "/internal/maintenance/status"): + return compose.Result{Stdout: fmt.Sprintf(`{"active":%t,"admissions":0,"recoveryRequired":false}`, runner.maintenance)}, nil + case strings.Contains(command, "/internal/maintenance/activate"): + runner.maintenance = true + return compose.Result{Stdout: `{"active":true,"admissions":0,"recoveryRequired":false}`}, nil + case strings.Contains(command, "/internal/maintenance/deactivate"): + runner.maintenance = false + return compose.Result{Stdout: `{"active":false,"admissions":0,"recoveryRequired":false}`}, nil + case strings.Contains(command, "/sessions?scope="): + if len(runner.sessionResponses) == 0 { + return compose.Result{Stdout: `[]`}, nil + } + response := runner.sessionResponses[0] + if len(runner.sessionResponses) > 1 { + runner.sessionResponses = runner.sessionResponses[1:] + } + return compose.Result{Stdout: response}, nil + case strings.HasSuffix(command, " stop"): + runner.stopCount++ + runner.running, runner.coreRunning = false, false + return compose.Result{}, nil + case strings.HasSuffix(command, " start"): + runner.startCount++ + runner.running, runner.coreRunning = true, true + return compose.Result{}, nil + case strings.Contains(command, " ps --all --format json"): + runner.healthChecks++ + return compose.Result{Stdout: healthyServicesPayload()}, nil + default: + return compose.Result{}, fmt.Errorf("unexpected fake Docker command: %s", command) + } +} + +func (runner *fakeBackupRunner) Stream(_ context.Context, args []string, _ io.Reader, stdout io.Writer) (compose.Result, error) { + runner.calls = append(runner.calls, strings.Join(args, " ")) + runner.streams++ + if runner.running { + runner.streamWhileRunning = true + } + writer := tar.NewWriter(stdout) + payload := []byte(fmt.Sprintf("volume-%d", runner.streams)) + if err := writer.WriteHeader(&tar.Header{Name: "payload", Mode: 0o600, Size: int64(len(payload))}); err != nil { + return compose.Result{}, err + } + if _, err := writer.Write(payload); err != nil { + return compose.Result{}, err + } + if err := writer.Close(); err != nil { + return compose.Result{}, err + } + if runner.failStreamAt == runner.streams { + return compose.Result{ExitCode: 1}, errors.New("fixture volume stream failed") + } + if runner.exitOnlyAt == runner.streams { + return compose.Result{ExitCode: 1}, nil + } + return compose.Result{}, nil +} + +func (runner *fakeBackupRunner) SessionInventoryScope() string { + if runner.installation.Profile == "local" { + return "mine" + } + return "all" +} + +func testDependencies(t *testing.T, runner archiveRunner) dependencies { + t.Helper() + return dependencies{ + runner: runner, + now: func() time.Time { return time.Date(2026, 8, 16, 8, 11, 12, 0, time.UTC) }, + homeDir: func() (string, error) { return t.TempDir(), nil }, + revision: func(context.Context, string) (string, error) { return testRevision, nil }, + sleep: func(time.Duration) {}, + rename: os.Rename, + } +} + +type fixtureArchive struct { + order []string + files map[string][]byte + manifest Manifest +} + +func readFixtureArchive(t *testing.T, path string) fixtureArchive { + t.Helper() + reader, err := zip.OpenReader(path) + if err != nil { + t.Fatal(err) + } + defer reader.Close() + result := fixtureArchive{files: make(map[string][]byte)} + for _, file := range reader.File { + result.order = append(result.order, file.Name) + opened, err := file.Open() + if err != nil { + t.Fatal(err) + } + contents, err := io.ReadAll(opened) + closeErr := opened.Close() + if err != nil || closeErr != nil { + t.Fatalf("read %s: %v / %v", file.Name, err, closeErr) + } + result.files[file.Name] = contents + } + if err := json.Unmarshal(result.files[ManifestPath], &result.manifest); err != nil { + t.Fatalf("decode manifest: %v", err) + } + return result +} + +func mapValues(values map[string][]byte) [][]byte { + keys := make([]string, 0, len(values)) + for key := range values { + keys = append(keys, key) + } + sort.Strings(keys) + result := make([][]byte, 0, len(keys)) + for _, key := range keys { + result = append(result, values[key]) + } + return result +} + +func activeSessionPayload() string { + return `[{"status":"running","archived":false}]` +} + +func healthyServicesPayload() string { + return strings.Join([]string{ + `{"Service":"core","State":"running","Health":"healthy"}`, + `{"Service":"frontend","State":"running","Health":"healthy"}`, + `{"Service":"qdrant","State":"running","Health":"healthy"}`, + `{"Service":"embedding","State":"running","Health":"healthy"}`, + `{"Service":"embedding-model-init","State":"exited","ExitCode":0}`, + }, "\n") +} + +func assertNoBackupArtifacts(t *testing.T, directory string) { + t.Helper() + entries, err := os.ReadDir(directory) + if err != nil { + t.Fatal(err) + } + if len(entries) != 0 { + names := make([]string, 0, len(entries)) + for _, entry := range entries { + names = append(names, entry.Name()) + } + t.Fatalf("incomplete backup artifacts remain: %v", names) + } +} diff --git a/tools/tht/internal/backup/manifest.go b/tools/tht/internal/backup/manifest.go new file mode 100644 index 00000000..50191cff --- /dev/null +++ b/tools/tht/internal/backup/manifest.go @@ -0,0 +1,215 @@ +package backup + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "path" + "regexp" + "sort" + "strings" + "time" +) + +const ( + CurrentSchemaVersion = 1 + ManifestPath = "manifest.json" + + EntryFile = "file" + EntryVolume = "volume" + EntrySecretReference = "external-secret-reference" + EntryExternalSecret = "external-secret" + EntryPreservationReference = "preservation-root-reference" +) + +var ( + checksumPattern = regexp.MustCompile(`^sha256:[0-9a-f]{64}$`) + revisionPattern = regexp.MustCompile(`^[0-9a-f]{40}([0-9a-f]{24})?$`) +) + +// Entry describes one logical backup payload or one external prerequisite. +type Entry struct { + Path string `json:"path"` + Kind string `json:"kind"` + Owner string `json:"owner"` + LogicalName string `json:"logical_name,omitempty"` + SourcePath string `json:"source_path,omitempty"` + SHA256 string `json:"sha256"` + Size int64 `json:"size"` + Mode uint32 `json:"mode,omitempty"` + Archived bool `json:"archived"` + Sensitive bool `json:"sensitive,omitempty"` +} + +// ImageIdentity records the configured image reference and, when available, the local image ID. +type ImageIdentity struct { + Service string `json:"service"` + Reference string `json:"reference"` + ID string `json:"id,omitempty"` +} + +// VolumeMetadata binds a logical Compose volume to its installation-owned Docker identity. +type VolumeMetadata struct { + LogicalName string `json:"logical_name"` + Name string `json:"name"` + Driver string `json:"driver"` + Labels map[string]string `json:"labels,omitempty"` +} + +// Manifest is the versioned restore contract written as the last archive member. +type Manifest struct { + SchemaVersion int `json:"schema_version"` + InstallationID string `json:"installation_id"` + CreatedAt time.Time `json:"created_at"` + SourceRevision string `json:"source_revision"` + IncludesSecrets bool `json:"includes_secrets"` + ComposeProject string `json:"compose_project"` + Images []ImageIdentity `json:"images"` + Volumes []VolumeMetadata `json:"volumes"` + Entries []Entry `json:"entries"` +} + +// DigestBytes returns the manifest's canonical checksum representation. +func DigestBytes(value []byte) string { + digest := sha256.Sum256(value) + return "sha256:" + hex.EncodeToString(digest[:]) +} + +// Finalize normalizes archive paths, orders every repeated field and validates the schema. +func (manifest *Manifest) Finalize() error { + manifest.CreatedAt = manifest.CreatedAt.UTC() + for index := range manifest.Entries { + normalized, err := normalizeArchivePath(manifest.Entries[index].Path) + if err != nil { + return err + } + manifest.Entries[index].Path = normalized + } + sort.Slice(manifest.Entries, func(left, right int) bool { + if manifest.Entries[left].Path != manifest.Entries[right].Path { + return manifest.Entries[left].Path < manifest.Entries[right].Path + } + if manifest.Entries[left].Kind != manifest.Entries[right].Kind { + return manifest.Entries[left].Kind < manifest.Entries[right].Kind + } + return manifest.Entries[left].Owner < manifest.Entries[right].Owner + }) + sort.Slice(manifest.Images, func(left, right int) bool { return manifest.Images[left].Service < manifest.Images[right].Service }) + sort.Slice(manifest.Volumes, func(left, right int) bool { + return manifest.Volumes[left].LogicalName < manifest.Volumes[right].LogicalName + }) + return manifest.Validate() +} + +// Validate rejects unsupported or unsafe restore contracts. +func (manifest Manifest) Validate() error { + if manifest.SchemaVersion != CurrentSchemaVersion { + return fmt.Errorf("unsupported backup manifest schema version %d", manifest.SchemaVersion) + } + if strings.TrimSpace(manifest.InstallationID) == "" || strings.ContainsAny(manifest.InstallationID, `/\\`) { + return errors.New("backup manifest installation ID is invalid") + } + if manifest.CreatedAt.IsZero() || manifest.CreatedAt.Location() != time.UTC { + return errors.New("backup manifest creation time must be UTC") + } + if !revisionPattern.MatchString(manifest.SourceRevision) { + return errors.New("backup manifest source revision is invalid") + } + if strings.TrimSpace(manifest.ComposeProject) == "" { + return errors.New("backup manifest Compose project is missing") + } + seenPaths := make(map[string]struct{}, len(manifest.Entries)) + includedExternalSecrets := 0 + for _, entry := range manifest.Entries { + if _, err := normalizeArchivePath(entry.Path); err != nil { + return err + } + if _, exists := seenPaths[entry.Path]; exists { + return fmt.Errorf("backup manifest contains duplicate entry path %q", entry.Path) + } + seenPaths[entry.Path] = struct{}{} + if entry.Owner == "" || entry.Kind == "" || entry.Size < 0 || !checksumPattern.MatchString(entry.SHA256) { + return fmt.Errorf("backup manifest entry %q is invalid", entry.Path) + } + if (entry.Kind == EntrySecretReference || entry.Kind == EntryExternalSecret) && entry.SourcePath == "" { + return fmt.Errorf("backup manifest external secret entry %q has no source path", entry.Path) + } + if entry.Kind == EntryExternalSecret { + if !entry.Archived || !entry.Sensitive { + return fmt.Errorf("backup manifest external secret entry %q is not marked sensitive and archived", entry.Path) + } + includedExternalSecrets++ + } + } + if manifest.IncludesSecrets != (includedExternalSecrets > 0) { + return errors.New("backup manifest external secret marker does not match included external secret payloads") + } + seenImages := make(map[string]struct{}, len(manifest.Images)) + for _, image := range manifest.Images { + if image.Service == "" || image.Reference == "" { + return errors.New("backup manifest image identity is incomplete") + } + if _, exists := seenImages[image.Service]; exists { + return fmt.Errorf("backup manifest contains duplicate image service %q", image.Service) + } + seenImages[image.Service] = struct{}{} + } + seenVolumes := make(map[string]struct{}, len(manifest.Volumes)) + for _, volume := range manifest.Volumes { + if volume.LogicalName == "" || volume.Name == "" || volume.Driver == "" { + return errors.New("backup manifest volume metadata is incomplete") + } + if _, exists := seenVolumes[volume.LogicalName]; exists { + return fmt.Errorf("backup manifest contains duplicate volume %q", volume.LogicalName) + } + seenVolumes[volume.LogicalName] = struct{}{} + } + return nil +} + +// JSON returns a deterministic, indented representation after validating a copy. +func (manifest Manifest) JSON() ([]byte, error) { + manifest.Entries = append([]Entry(nil), manifest.Entries...) + manifest.Images = append([]ImageIdentity(nil), manifest.Images...) + manifest.Volumes = append([]VolumeMetadata(nil), manifest.Volumes...) + if err := manifest.Finalize(); err != nil { + return nil, err + } + value, err := json.MarshalIndent(manifest, "", " ") + if err != nil { + return nil, err + } + return append(value, '\n'), nil +} + +// DecodeManifest decodes exactly one supported manifest document. +func DecodeManifest(value []byte) (Manifest, error) { + decoder := json.NewDecoder(bytes.NewReader(value)) + decoder.DisallowUnknownFields() + var manifest Manifest + if err := decoder.Decode(&manifest); err != nil { + return Manifest{}, fmt.Errorf("decode backup manifest: %w", err) + } + if decoder.More() { + return Manifest{}, errors.New("backup manifest contains trailing data") + } + if err := manifest.Finalize(); err != nil { + return Manifest{}, err + } + return manifest, nil +} + +func normalizeArchivePath(value string) (string, error) { + value = strings.ReplaceAll(value, `\`, "/") + if value == "" || strings.HasPrefix(value, "/") { + return "", errors.New("backup manifest entry path is empty or absolute") + } + normalized := path.Clean(value) + if normalized == "." || normalized == ".." || strings.HasPrefix(normalized, "../") { + return "", fmt.Errorf("backup manifest entry path %q escapes the archive", value) + } + return normalized, nil +} diff --git a/tools/tht/internal/backup/manifest_test.go b/tools/tht/internal/backup/manifest_test.go new file mode 100644 index 00000000..80fa069d --- /dev/null +++ b/tools/tht/internal/backup/manifest_test.go @@ -0,0 +1,131 @@ +package backup + +import ( + "encoding/json" + "strings" + "testing" + "time" +) + +const testRevision = "0123456789abcdef0123456789abcdef01234567" + +func TestManifestFinalizationNormalizesAndOrdersEntriesDeterministically(t *testing.T) { + manifest := Manifest{ + SchemaVersion: CurrentSchemaVersion, + InstallationID: "local-dev", + CreatedAt: time.Date(2026, 8, 16, 10, 11, 12, 123, time.FixedZone("test", 2*60*60)), + SourceRevision: testRevision, + ComposeProject: "thothii-test", + Entries: []Entry{ + {Path: `volumes\sessions.tar`, Kind: EntryVolume, Owner: "volume:sessions", SHA256: DigestBytes([]byte("sessions")), Size: 8, Archived: true}, + {Path: "configuration/pi/models.json", Kind: EntryFile, Owner: "pi", SHA256: DigestBytes([]byte("models")), Size: 6, Archived: true}, + }, + Images: []ImageIdentity{ + {Service: "frontend", Reference: "frontend:test", ID: "sha256:front"}, + {Service: "core", Reference: "core:test", ID: "sha256:core"}, + }, + Volumes: []VolumeMetadata{ + {LogicalName: "sessions", Name: "thothii-test_sessions", Driver: "local"}, + {LogicalName: "pi-state", Name: "thothii-test_pi-state", Driver: "local"}, + }, + } + + if err := manifest.Finalize(); err != nil { + t.Fatal(err) + } + if got, want := manifest.CreatedAt, time.Date(2026, 8, 16, 8, 11, 12, 123, time.UTC); !got.Equal(want) || got.Location() != time.UTC { + t.Fatalf("CreatedAt = %v (%v), want %v UTC", got, got.Location(), want) + } + if got := []string{manifest.Entries[0].Path, manifest.Entries[1].Path}; got[0] != "configuration/pi/models.json" || got[1] != "volumes/sessions.tar" { + t.Fatalf("entry order = %v", got) + } + if manifest.Images[0].Service != "core" || manifest.Volumes[0].LogicalName != "pi-state" { + t.Fatalf("metadata was not deterministically ordered: images=%v volumes=%v", manifest.Images, manifest.Volumes) + } + first, err := manifest.JSON() + if err != nil { + t.Fatal(err) + } + second, err := manifest.JSON() + if err != nil { + t.Fatal(err) + } + if string(first) != string(second) { + t.Fatalf("manifest encoding is not deterministic:\n%s\n%s", first, second) + } +} + +func TestManifestRejectsUnsafePathsInvalidChecksumsAndUnsupportedSchemas(t *testing.T) { + valid := Manifest{ + SchemaVersion: CurrentSchemaVersion, + InstallationID: "local-dev", + CreatedAt: time.Now().UTC(), + SourceRevision: testRevision, + ComposeProject: "thothii-test", + Entries: []Entry{{Path: "configuration/operator.env", Kind: EntryFile, Owner: "installation", SHA256: DigestBytes(nil), Archived: true}}, + } + + for _, mutate := range []func(*Manifest){ + func(value *Manifest) { value.SchemaVersion = CurrentSchemaVersion + 1 }, + func(value *Manifest) { value.Entries[0].Path = "../outside" }, + func(value *Manifest) { value.Entries[0].Path = "/absolute" }, + func(value *Manifest) { value.Entries[0].SHA256 = "sha256:not-a-digest" }, + func(value *Manifest) { value.Entries = append(value.Entries, value.Entries[0]) }, + } { + candidate := valid + candidate.Entries = append([]Entry(nil), valid.Entries...) + mutate(&candidate) + if err := candidate.Finalize(); err == nil { + t.Fatalf("Finalize() accepted invalid manifest: %#v", candidate) + } + } +} + +func TestManifestSecretMarkerRequiresIncludedExternalSecretPayload(t *testing.T) { + base := Manifest{ + SchemaVersion: CurrentSchemaVersion, + InstallationID: "local-dev", + CreatedAt: time.Now().UTC(), + SourceRevision: testRevision, + ComposeProject: "thothii-test", + Entries: []Entry{ + {Path: "volumes/workspace-secrets.tar", Kind: EntryVolume, Owner: "volume:workspace-secrets", SHA256: DigestBytes([]byte("managed")), Size: 7, Archived: true, Sensitive: true}, + {Path: "external-secrets/000", Kind: EntrySecretReference, Owner: "external-secret", SHA256: DigestBytes([]byte("external")), Size: 8, Archived: false, Sensitive: true, SourcePath: "/protected/secret"}, + }, + } + if err := base.Finalize(); err != nil { + t.Fatalf("managed workspace secrets and an excluded external reference must be valid: %v", err) + } + + base.IncludesSecrets = true + if err := base.Finalize(); err == nil || !strings.Contains(err.Error(), "external secret") { + t.Fatalf("Finalize() error = %v, want missing included external secret", err) + } + for index := range base.Entries { + if base.Entries[index].Kind == EntrySecretReference { + base.Entries[index].Kind = EntryExternalSecret + base.Entries[index].Archived = true + base.Entries[index].Path = "external-secrets/000-secret" + } + } + if err := base.Finalize(); err != nil { + t.Fatalf("included external secret marker was rejected: %v", err) + } +} + +func TestDecodeManifestRejectsASecondDocument(t *testing.T) { + manifest := Manifest{ + SchemaVersion: CurrentSchemaVersion, + InstallationID: "local-dev", + CreatedAt: time.Date(2026, 8, 16, 8, 11, 12, 0, time.UTC), + SourceRevision: testRevision, + ComposeProject: "thothii-test", + } + encoded, err := json.Marshal(manifest) + if err != nil { + t.Fatal(err) + } + if _, err := DecodeManifest(append(encoded, encoded...)); err == nil { + t.Fatal("DecodeManifest() accepted a second JSON document") + } +} diff --git a/tools/tht/internal/lifecycle/lock.go b/tools/tht/internal/lifecycle/lock.go new file mode 100644 index 00000000..f919e32b --- /dev/null +++ b/tools/tht/internal/lifecycle/lock.go @@ -0,0 +1,113 @@ +// Package lifecycle coordinates installation-wide mutating operations. +package lifecycle + +import ( + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "sync" + "time" + + "github.com/aritmolab/thothii/tools/tht/internal/config" +) + +var ( + ErrLocked = errors.New("another lifecycle operation is already running for this installation") + ErrOwnership = errors.New("lifecycle lock ownership changed; refusing to remove it") +) + +const lockFileName = "lifecycle.lock.owner.json" + +type owner struct { + Token string `json:"token"` + PID int `json:"pid"` + CreatedAt time.Time `json:"created_at"` +} + +// Lock is an exclusively-created owner file scoped to one installation descriptor. +type Lock struct { + path string + token string + mu sync.Mutex + released bool +} + +// Acquire obtains the shared lock used by backup, restore, Pi lifecycle and product updates. +func Acquire(installation config.Installation) (*Lock, error) { + directory := installation.ControlDirectory() + if err := os.MkdirAll(directory, 0o700); err != nil { + return nil, fmt.Errorf("create lifecycle control directory: %w", err) + } + info, err := os.Lstat(directory) + if err != nil || !info.IsDir() || info.Mode()&os.ModeSymlink != 0 { + return nil, errors.New("lifecycle control directory is not a regular directory") + } + if err := os.Chmod(directory, 0o700); err != nil { + return nil, fmt.Errorf("protect lifecycle control directory: %w", err) + } + + tokenBytes := make([]byte, 16) + if _, err := rand.Read(tokenBytes); err != nil { + return nil, fmt.Errorf("create lifecycle lock owner: %w", err) + } + token := hex.EncodeToString(tokenBytes) + path := filepath.Join(directory, lockFileName) + file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) + if errors.Is(err, os.ErrExist) { + return nil, ErrLocked + } + if err != nil { + return nil, fmt.Errorf("acquire lifecycle lock: %w", err) + } + value := owner{Token: token, PID: os.Getpid(), CreatedAt: time.Now().UTC()} + encodeErr := json.NewEncoder(file).Encode(value) + if encodeErr == nil { + encodeErr = file.Sync() + } + closeErr := file.Close() + if encodeErr != nil || closeErr != nil { + _ = os.Remove(path) + return nil, fmt.Errorf("persist lifecycle lock owner: %w", errors.Join(encodeErr, closeErr)) + } + return &Lock{path: path, token: token}, nil +} + +// Path returns the installation-private owner-file path for diagnostics and tests. +func (lock *Lock) Path() string { + if lock == nil { + return "" + } + return lock.path +} + +// Release removes only the owner file created by this Lock. +func (lock *Lock) Release() error { + if lock == nil { + return nil + } + lock.mu.Lock() + defer lock.mu.Unlock() + if lock.released { + return nil + } + contents, err := os.ReadFile(lock.path) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return ErrOwnership + } + return fmt.Errorf("read lifecycle lock owner: %w", err) + } + var current owner + if json.Unmarshal(contents, ¤t) != nil || current.Token == "" || current.Token != lock.token { + return ErrOwnership + } + if err := os.Remove(lock.path); err != nil { + return fmt.Errorf("release lifecycle lock: %w", err) + } + lock.released = true + return nil +} diff --git a/tools/tht/internal/lifecycle/lock_test.go b/tools/tht/internal/lifecycle/lock_test.go new file mode 100644 index 00000000..196e97ef --- /dev/null +++ b/tools/tht/internal/lifecycle/lock_test.go @@ -0,0 +1,50 @@ +package lifecycle + +import ( + "errors" + "os" + "path/filepath" + "testing" + + "github.com/aritmolab/thothii/tools/tht/internal/config" +) + +func TestLifecycleLockIsExclusivePerInstallationAndReusableAfterRelease(t *testing.T) { + installation := config.Installation{ProjectDirectory: t.TempDir(), Path: filepath.Join(t.TempDir(), "thothii-installation.yaml")} + first, err := Acquire(installation) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = first.Release() }) + + if _, err := Acquire(installation); !errors.Is(err, ErrLocked) { + t.Fatalf("second Acquire() error = %v, want ErrLocked", err) + } + if err := first.Release(); err != nil { + t.Fatal(err) + } + second, err := Acquire(installation) + if err != nil { + t.Fatalf("Acquire() after release: %v", err) + } + if err := second.Release(); err != nil { + t.Fatal(err) + } +} + +func TestLifecycleLockReleaseDoesNotRemoveAnotherOwnersFile(t *testing.T) { + installation := config.Installation{ProjectDirectory: t.TempDir(), Path: filepath.Join(t.TempDir(), "thothii-installation.yaml")} + lock, err := Acquire(installation) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(lock.Path(), []byte(`{"token":"different-owner"}`), 0o600); err != nil { + t.Fatal(err) + } + if err := lock.Release(); !errors.Is(err, ErrOwnership) { + t.Fatalf("Release() error = %v, want ErrOwnership", err) + } + if _, err := os.Stat(lock.Path()); err != nil { + t.Fatalf("foreign lock was removed: %v", err) + } +}