diff --git a/raft/src/main/java/org/apache/kafka/raft/VoterSet.java b/raft/src/main/java/org/apache/kafka/raft/VoterSet.java
index 23f968cdeeb58..1c0c67e31c33b 100644
--- a/raft/src/main/java/org/apache/kafka/raft/VoterSet.java
+++ b/raft/src/main/java/org/apache/kafka/raft/VoterSet.java
@@ -47,9 +47,13 @@
  */
 public final class VoterSet {
     private final Map<Integer, VoterNode> voters;
+    private final Set<ReplicaKey> voterKeys;
 
     private VoterSet(Map<Integer, VoterNode> voters) {
         this.voters = voters;
+        this.voterKeys = voters.values().stream()
+            .map(VoterNode::voterKey)
+            .collect(Collectors.toUnmodifiableSet());
     }
 
     /**
@@ -144,11 +148,7 @@ public Set<Integer> voterIds() {
      * Returns all of the voters.
      */
     public Set<ReplicaKey> voterKeys() {
-        return voters
-            .values()
-            .stream()
-            .map(VoterNode::voterKey)
-            .collect(Collectors.toSet());
+        return voterKeys;
     }
 
     /**