fix(auth): recover interrupted projection retention

tools/tht/internal/authprojection/projection_linux.go

@@ -93,7 +93,16 @@ func Begin(spec Spec, before *Snapshot, requireReadyMatch bool) (*Transaction, e
 			_ = transaction.Close()
 			return nil, ErrIntegrity
 		}
-		status, inspectErr := inspectCapturedSelector(root, spec, selector)
+		status, inspectErr := inspectSelectedGeneration(root, spec, selector)
+		if inspectErr != nil {
+			_ = transaction.Close()
+			return nil, ErrIntegrity
+		}
+		if validateReadyGenerationHistory(root, spec, selector) != nil || retain(root, spec, selector) != nil {
+			_ = transaction.Close()
+			return nil, ErrIntegrity
+		}
+		status, inspectErr = inspectCapturedSelector(root, spec, selector)
 		if inspectErr != nil {
 			_ = transaction.Close()
 			return nil, ErrIntegrity
@@ -372,6 +381,9 @@ func inspectSelectedGeneration(root int, spec Spec, selector Selector) (Status,
 }
 
 func validateReadyGenerationNamespace(root int, spec Spec, selector Selector) error {
+	if validateReadyGenerationHistory(root, spec, selector) != nil {
+		return ErrIntegrity
+	}
 	generations, err := openDirectoryAt(root, "generations", spec)
 	if err != nil {
 		return ErrIntegrity
@@ -392,7 +404,16 @@ func validateReadyGenerationNamespace(root int, spec Spec, selector Selector) er
 		if !validHex(name, 64) || !keep[name] {
 			return ErrIntegrity
 		}
-		if _, err := readGeneration(root, spec, name); err != nil {
+	}
+	return nil
+}
+
+func validateReadyGenerationHistory(root int, spec Spec, selector Selector) error {
+	if selector.State != "ready" {
+		return ErrIntegrity
+	}
+	for _, generation := range append([]string{selector.Generation}, selector.PreviousGenerations...) {
+		if _, err := readGeneration(root, spec, generation); err != nil {
 			return ErrIntegrity
 		}
 	}
@@ -652,6 +673,9 @@ func retain(root int, spec Spec, selector Selector) error {
 			}
 			continue
 		}
+		if _, err := readGeneration(root, spec, name); err != nil {
+			return ErrIntegrity
+		}
 		if removeConfinedDirectory(generations, name, spec, false) != nil {
 			return ErrIntegrity
 		}

tools/tht/internal/authprojection/projection_linux_test.go

@@ -555,6 +555,52 @@ func TestRetentionKeepsCurrentAndTwoPredecessors(t *testing.T) {
 	}
 }
 
+func TestBeginCompletesSafeRetentionAfterReadyPublicationInterruptedBeforeCleanup(t *testing.T) {
+	spec := testSpec(t)
+	var snapshots []Snapshot
+	for index := 0; index < 3; index++ {
+		snapshot := testSnapshot(t, fmt.Sprintf("interrupted-%d", index))
+		commitSnapshot(t, spec, snapshot)
+		snapshots = append(snapshots, snapshot)
+	}
+	fourth := testSnapshot(t, "interrupted-3")
+	transaction, err := Begin(spec, nil, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	restore := setTestHooksForTest(testHooks{beforeRetention: func() error { return errors.New("sentinel") }})
+	if _, err := transaction.Commit(fourth); !errors.Is(err, ErrIntegrity) || strings.Contains(fmt.Sprint(err), "sentinel") {
+		t.Fatalf("interrupted Commit error = %v", err)
+	}
+	restore()
+	if err := transaction.Close(); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := Inspect(spec); !errors.Is(err, ErrIntegrity) {
+		t.Fatalf("Inspect after interrupted retention error = %v", err)
+	}
+
+	fifth := testSnapshot(t, "interrupted-4")
+	recovery, err := Begin(spec, &fourth, true)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer recovery.Close()
+	status, err := recovery.Commit(fifth)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if status.Snapshot.Generation != fifth.Generation {
+		t.Fatalf("generation = %s", status.Snapshot.Generation)
+	}
+	if _, err := Inspect(spec); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := os.Lstat(filepath.Join(spec.RuntimeRoot, "generations", snapshots[0].Generation)); !errors.Is(err, os.ErrNotExist) {
+		t.Fatalf("surplus generation remains: %v", err)
+	}
+}
+
 func TestConcurrentTransactionsNeverPublishMixedGeneration(t *testing.T) {
 	spec := testSpec(t)
 	first := testSnapshot(t, "one")