mptyl/ThothII
1
0
Fork 0
Code Issues 25 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

feat(server): activate projected authentication safely

Browse Source
This commit is contained in:
User
2026-08-22 01:01:36 +02:00
parent 903c0b4de5
commit 3d9a9f0675
32 changed files with 1837 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/tht/internal/backup/restore_test.go
+253 -18
View File
@@ -13,6 +13,7 @@ import (
"testing"
"time"
"github.com/aritmolab/thothii/tools/tht/internal/authconfig"
"github.com/aritmolab/thothii/tools/tht/internal/compose"
"github.com/aritmolab/thothii/tools/tht/internal/config"
"github.com/aritmolab/thothii/tools/tht/internal/lifecycle"
@@ -624,7 +625,7 @@ func TestRestoreLifecycleLockExcludesCompetingTransactionsUntilTerminalCleanup(t
}
return targetFailure
}
deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool) error {
deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool, authProjectionRestoreTransaction) error {
if err := gate("recovery"); err != nil {
return err
}
@@ -645,7 +646,7 @@ func TestRestoreLifecycleLockExcludesCompetingTransactionsUntilTerminalCleanup(t
}
return targetFailure
}
deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool) error {
deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool, authProjectionRestoreTransaction) error {
if err := gate("recovery-failure"); err != nil {
return err
}
@@ -666,7 +667,7 @@ func TestRestoreLifecycleLockExcludesCompetingTransactionsUntilTerminalCleanup(t
cancel()
return context.Canceled
}
deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool) error {
deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool, authProjectionRestoreTransaction) error {
if err := gate("recovery"); err != nil {
return err
}
@@ -892,7 +893,7 @@ func TestRestoreCannotApplyAStaleCheckpointOverAnInterleavedRestore(t *testing.T
firstDeps.restoreFile = func(context.Context, config.Installation, ArchiveEntryMetadata, io.Reader) error {
return errors.New("first target mutation failed before changing state")
}
firstDeps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool) error {
firstDeps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool, authProjectionRestoreTransaction) error {
recoveryObserved = targetState
targetState = checkpointState
firstRunner.running, firstRunner.coreRunning = true, true
@@ -1095,7 +1096,7 @@ func TestRestoreFileFailureRollsBackSecretAwareCheckpointBeforeCleanup(t *testin
return Result{Path: "/tmp/recovery.zip"}, nil
}
var events []string
deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool) error {
deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool, authProjectionRestoreTransaction) error {
events = append(events, "recover")
return nil
}
@@ -1135,7 +1136,7 @@ func TestRestoreFailureAfterAuthenticationMutationRollsBackAndClearsRuntimeState
events = append(events, "auth-runtime-reset-failed")
return resetErr
}
deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool) error {
deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool, authProjectionRestoreTransaction) error {
events = append(events, "secret-aware-recovery-and-reauth-reset")
return nil
}
@@ -1161,7 +1162,7 @@ func TestRestoreCleanupFailureDoesNotSuppressRollback(t *testing.T) {
cleanupErr := errors.New("checkpoint cleanup failure")
recovered := false
deps.restoreFile = func(context.Context, config.Installation, ArchiveEntryMetadata, io.Reader) error { return mutationErr }
deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool) error {
deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool, authProjectionRestoreTransaction) error {
recovered = true
return nil
}
@@ -1223,7 +1224,7 @@ func TestRestoreStartFailureRecoversPreviouslyRunningTarget(t *testing.T) {
backingRunner := newBackupRunner(installation, true)
deps := restoreTestDependencies(t, failStartRestoreRunner{fakeBackupRunner: backingRunner})
recovered := false
deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool) error {
deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool, authProjectionRestoreTransaction) error {
recovered = true
backingRunner.running = true
return nil
@@ -1246,7 +1247,7 @@ func TestRestoreVerificationFailureRecoversPreviouslyRunningTarget(t *testing.T)
verificationErr := errors.New("Pi is unavailable")
deps.verify["pi"] = func(context.Context, config.Installation, archiveRunner) error { return verificationErr }
recovered := false
deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool) error {
deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool, authProjectionRestoreTransaction) error {
recovered = true
return nil
}
@@ -1352,7 +1353,7 @@ func TestRestoreRecoversBehindBarrierForEveryVerificationFailure(t *testing.T) {
}
var recoveryBarrierActive bool
var recoveryContext cleanupContextObservation
deps.recover = func(ctx context.Context, _ config.Installation, _ PreflightResult, _ *stagedArchive, _ bool) error {
deps.recover = func(ctx context.Context, _ config.Installation, _ PreflightResult, _ *stagedArchive, _ bool, _ authProjectionRestoreTransaction) error {
recoveryContext = observeCleanupContext(ctx)
recoveryBarrierActive = runner.maintenance
runner.running, runner.coreRunning = true, true
@@ -1392,7 +1393,7 @@ func TestRestoreDoesNotRollbackAfterFinalDeactivationResponseLoss(t *testing.T)
}
deps := restoreTestDependencies(t, runner)
recoveryCalls := 0
deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool) error {
deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool, authProjectionRestoreTransaction) error {
recoveryCalls++
return nil
}
@@ -1450,7 +1451,7 @@ func TestRestoreUsesBoundedRecoveryContextAfterPostMutationCancellation(t *testi
}
var recoveryContext cleanupContextObservation
var recoveryBarrierActive bool
deps.recover = func(ctx context.Context, _ config.Installation, _ PreflightResult, _ *stagedArchive, _ bool) error {
deps.recover = func(ctx context.Context, _ config.Installation, _ PreflightResult, _ *stagedArchive, _ bool, _ authProjectionRestoreTransaction) error {
recoveryContext = observeCleanupContext(ctx)
recoveryBarrierActive = runner.maintenance
runner.running, runner.coreRunning = true, true
@@ -1536,7 +1537,7 @@ func TestRecoverRestoreTransactionVerifiesRecoveredStateBeforeReturning(t *testi
}
staged := stageRecoveryForTest(t, installation, recovery)
if err := recoverRestoreTransaction(context.Background(), installation, recovery, staged, true, deps); err != nil {
if err := recoverRestoreTransaction(context.Background(), installation, recovery, staged, true, deps, nil); err != nil {
t.Fatal(err)
}
if got, want := checks, []string{"health", "doctor", "pi", "workspace"}; !equalStrings(got, want) {
@@ -1566,7 +1567,7 @@ func TestRecoverRestoreTransactionFailsClosedForEveryVerification(t *testing.T)
}
staged := stageRecoveryForTest(t, installation, recovery)
err = recoverRestoreTransaction(context.Background(), installation, recovery, staged, true, deps)
err = recoverRestoreTransaction(context.Background(), installation, recovery, staged, true, deps, nil)
if !errors.Is(err, verificationErr) {
t.Fatalf("recoverRestoreTransaction() error = %v, want %v", err, verificationErr)
}
@@ -1646,8 +1647,8 @@ func TestRestoreReleasesBarrierOnlyAfterVerifiedRecoveryFromLostResponse(t *test
}
return nil
}
deps.recover = func(ctx context.Context, target config.Installation, checkpoint PreflightResult, staged *stagedArchive, wasRunning bool) error {
return recoverRestoreTransaction(ctx, target, checkpoint, staged, wasRunning, deps)
deps.recover = func(ctx context.Context, target config.Installation, checkpoint PreflightResult, staged *stagedArchive, wasRunning bool, transaction authProjectionRestoreTransaction) error {
return recoverRestoreTransaction(ctx, target, checkpoint, staged, wasRunning, deps, transaction)
}
_, err = restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: archive, Confirm: true}, deps)
@@ -1744,7 +1745,7 @@ func TestRestoreCleansMaintenanceAfterMutationAndRollbackFailures(t *testing.T)
deps.restoreFile = func(context.Context, config.Installation, ArchiveEntryMetadata, io.Reader) error {
return mutationErr
}
deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool) error {
deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool, authProjectionRestoreTransaction) error {
if test.recoveryErr == nil {
backing.running, backing.coreRunning = true, true
}
@@ -2005,7 +2006,9 @@ func restoreTestDependencies(t *testing.T, runner archiveRunner) restoreDependen
prepareRecovery: func(ctx context.Context, installation config.Installation, _ string) (PreflightResult, error) {
return Preflight(ctx, installation, PreflightRequest{Archive: recoveryArchive, Confirm: true, AllowExternalSecrets: true}, permissivePreflightDependencies())
},
recover: func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool) error { return nil },
recover: func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool, authProjectionRestoreTransaction) error {
return nil
},
cleanupCheckpoint: func(string) error { return nil },
acquireTransaction: lifecycle.AcquireTransaction,
runner: runner,
@@ -2025,3 +2028,235 @@ func restoreTestDependencies(t *testing.T, runner archiveRunner) restoreDependen
},
}
}
type projectionRestoreStub struct {
events *[]string
blocked bool
publishErr error
restoreErr error
closeErr error
}
func (stub *projectionRestoreStub) PublishCanonical() (authconfig.ProjectionStatus, error) {
*stub.events = append(*stub.events, "publish")
if stub.publishErr != nil {
return authconfig.ProjectionStatus{}, stub.publishErr
}
stub.blocked = false
return authconfig.ProjectionStatus{State: "ready", Generation: "g", CanonicalRevision: "sha256:g", Equal: true}, nil
}
func (stub *projectionRestoreStub) RestorePriorIfCanonicalUnchanged() error {
*stub.events = append(*stub.events, "restore-prior")
if stub.restoreErr != nil {
return stub.restoreErr
}
stub.blocked = false
return nil
}
func (stub *projectionRestoreStub) Close() error {
*stub.events = append(*stub.events, "close")
return stub.closeErr
}
type restartEventRunner struct {
archiveRunner
events *[]string
}
func (runner restartEventRunner) Run(ctx context.Context, args []string, input io.Reader) (compose.Result, error) {
if strings.HasSuffix(strings.Join(args, " "), " start") {
*runner.events = append(*runner.events, "restart")
}
return runner.archiveRunner.Run(ctx, args, input)
}
func (runner restartEventRunner) Stream(ctx context.Context, args []string, input io.Reader, output io.Writer) (compose.Result, error) {
return runner.archiveRunner.Stream(ctx, args, input, output)
}
func (runner restartEventRunner) SessionInventoryScope() string {
return runner.archiveRunner.SessionInventoryScope()
}
func projectedRestoreFixture(t *testing.T) (config.Installation, string) {
t.Helper()
installation := preflightTestInstallation(t)
authRoot := filepath.Join(t.TempDir(), "canonical-auth")
if err := os.Mkdir(authRoot, 0o700); err != nil {
t.Fatal(err)
}
installation.Authentication.ConfigDirectory = authRoot
installation.Authentication.RuntimeProjection = &config.RuntimeProjection{Directory: filepath.Join(t.TempDir(), "runtime-auth"), UID: 10001, GID: 10001}
archive := filepath.Join(t.TempDir(), "auth-restore.zip")
writePreflightArchive(t, archive, preflightArchiveSpec{includeSecrets: true, entries: []preflightArchiveEntry{{
path: "authentication-secrets/000-auth.yaml", body: []byte("candidate auth\n"), kind: EntryExternalSecret,
sensitive: true, owner: "authentication-configuration", sourcePath: filepath.Join(authRoot, "auth.yaml"),
}}})
return installation, archive
}
func assertOrderedEvents(t *testing.T, events []string, wants ...string) {
t.Helper()
at := 0
for _, want := range wants {
for at < len(events) && events[at] != want {
at++
}
if at == len(events) {
t.Fatalf("events = %v, want ordered subsequence %v", events, wants)
}
at++
}
}
func TestRestoreAuthBearingArchiveBlocksBeforeWritePublishesBeforeRestart(t *testing.T) {
installation, archive := projectedRestoreFixture(t)
var events []string
backing := newBackupRunner(installation, true)
runner := restartEventRunner{archiveRunner: backing, events: &events}
deps := restoreTestDependencies(t, runner)
transaction := &projectionRestoreStub{events: &events}
deps.beginAuthProjection = func(context.Context, config.Installation) (authProjectionRestoreTransaction, error) {
transaction.blocked = true
events = append(events, "begin")
return transaction, nil
}
deps.restoreFile = func(_ context.Context, _ config.Installation, entry ArchiveEntryMetadata, _ io.Reader) error {
if entry.Owner == "authentication-configuration" {
if !transaction.blocked {
t.Fatal("auth destination write began without blocked projection")
}
events = append(events, "restore-auth")
}
return nil
}
if _, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: archive, Confirm: true}, deps); err != nil {
t.Fatal(err)
}
assertOrderedEvents(t, events, "begin", "restore-auth", "publish", "restart", "close")
if transaction.blocked {
t.Fatalf("successful restore closed while projection remained blocked: %v", events)
}
}
func TestRestoreAuthCandidatePublicationFailureRecoversThenStaysBlockedWithoutRestart(t *testing.T) {
installation, archive := projectedRestoreFixture(t)
var events []string
backing := newBackupRunner(installation, true)
runner := restartEventRunner{archiveRunner: backing, events: &events}
deps := restoreTestDependencies(t, runner)
publicationErr := errors.New("synthetic publication failure")
transaction := &projectionRestoreStub{events: &events, publishErr: publicationErr}
deps.beginAuthProjection = func(context.Context, config.Installation) (authProjectionRestoreTransaction, error) {
transaction.blocked = true
return transaction, nil
}
deps.recover = func(_ context.Context, _ config.Installation, _ PreflightResult, _ *stagedArchive, _ bool, transaction authProjectionRestoreTransaction) error {
events = append(events, "restore-checkpoint")
_, err := transaction.PublishCanonical()
return err
}
if _, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: archive, Confirm: true}, deps); !errors.Is(err, publicationErr) {
t.Fatalf("restore error = %v, want publication failure", err)
}
assertOrderedEvents(t, events, "publish", "restore-checkpoint", "close")
if !transaction.blocked {
t.Fatal("publication failure reopened projection")
}
if backing.startCount != 0 {
t.Fatalf("restart after failed candidate/recovery publication = %d", backing.startCount)
}
if !backing.maintenance {
t.Fatal("admissions reopened after failed recovery publication")
}
}
func TestRestoreAuthVerificationFailuresRepublishCheckpointBeforeRecoveryRestart(t *testing.T) {
for _, failed := range []string{"health", "doctor", "pi", "workspace"} {
t.Run(failed, func(t *testing.T) {
installation, archive := projectedRestoreFixture(t)
var events []string
backing := newBackupRunner(installation, true)
runner := restartEventRunner{archiveRunner: backing, events: &events}
deps := restoreTestDependencies(t, runner)
transaction := &projectionRestoreStub{events: &events}
deps.beginAuthProjection = func(context.Context, config.Installation) (authProjectionRestoreTransaction, error) {
transaction.blocked = true
return transaction, nil
}
for _, name := range []string{"health", "doctor", "pi", "workspace"} {
name := name
deps.verify[name] = func(context.Context, config.Installation, archiveRunner) error {
events = append(events, "candidate-"+name)
if name == failed {
return errors.New("synthetic " + name + " failure")
}
return nil
}
}
deps.recover = func(ctx context.Context, target config.Installation, _ PreflightResult, _ *stagedArchive, wasRunning bool, transaction authProjectionRestoreTransaction) error {
events = append(events, "restore-checkpoint")
if _, err := transaction.PublishCanonical(); err != nil {
return err
}
events = append(events, "verify-checkpoint")
if wasRunning {
return composeStartAndVerify(ctx, target, runner)
}
return nil
}
if _, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: archive, Confirm: true}, deps); err == nil {
t.Fatalf("%s failure accepted", failed)
}
assertOrderedEvents(t, events, "publish", "candidate-"+failed, "restore-checkpoint", "publish", "verify-checkpoint", "restart", "close")
if transaction.blocked {
t.Fatalf("verified checkpoint recovery remained blocked: %v", events)
}
})
}
}
func TestRestoreAuthPreMutationFailureRestoresPriorReadySelector(t *testing.T) {
installation, archive := projectedRestoreFixture(t)
var events []string
backing := newBackupRunner(installation, false)
runner := &commandFailureRunner{fakeBackupRunner: backing, failures: []*commandFailure{{
match: func(command string) bool { return strings.Contains(command, " ps --all --format json") },
err: errors.New("synthetic pre-mutation failure"), remaining: 1,
}}}
deps := restoreTestDependencies(t, runner)
transaction := &projectionRestoreStub{events: &events}
deps.beginAuthProjection = func(context.Context, config.Installation) (authProjectionRestoreTransaction, error) {
transaction.blocked = true
return transaction, nil
}
deps.recover = func(context.Context, config.Installation, PreflightResult, *stagedArchive, bool, authProjectionRestoreTransaction) error {
t.Fatal("recovery ran before any destination mutation")
return nil
}
if _, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: archive, Confirm: true}, deps); err == nil {
t.Fatal("pre-mutation failure accepted")
}
assertOrderedEvents(t, events, "restore-prior", "close")
if transaction.blocked {
t.Fatal("unchanged canonical pre-write failure did not restore ready selector")
}
}
func TestRestoreNonAuthArchiveNeverBeginsProjection(t *testing.T) {
installation := preflightTestInstallation(t)
deps := restoreTestDependencies(t, newBackupRunner(installation, false))
called := false
deps.beginAuthProjection = func(context.Context, config.Installation) (authProjectionRestoreTransaction, error) {
called = true
return nil, errors.New("must not begin")
}
if _, err := restoreWithDependencies(context.Background(), installation, RestoreRequest{Archive: restoreArchive(t), Confirm: true}, deps); err != nil {
t.Fatal(err)
}
if called {
t.Fatal("non-auth archive began projection transaction")
}
}