(2.14) [ADDED] Config reload: add/remove remote leafnodes (#7937)

The configuration reload now supports adding and/or removing remote
leafnodes. A remote is identified with the combination of its URLs list,
local account and credentials file name. This is what is used by the
server to detect changes for the remote leafnodes list.

Signed-off-by: Ivan Kozlovic <ivan@synadia.com>

Author: neilalexander

server/client.go

@@ -6189,7 +6189,7 @@ func (c *client) reconnect() {
 			srv.Debugf("Gateway %q not in configuration, not attempting reconnect", gwName)
 		}
 	} else if leafCfg != nil {
-		// Check if this is a solicited leaf node. Start up a reconnect.
+		// This is a solicited leaf node. Start up a reconnect.
 		srv.startGoRoutine(func() { srv.reConnectToRemoteLeafNode(leafCfg) })
 	}
 }

server/config_check_test.go

@@ -2025,6 +2025,22 @@ func TestConfigCheck(t *testing.T) {
 			errorLine: 7,
 			errorPos:  8,
 		},
+		{
+			name: "duplicate remote leafnode",
+			config: `
+				leafnodes {
+					port: -1
+					remotes [
+						{ url: "nats://user1:pwd@127.0.0.1:1234" }
+						{ url: "nats://user2:pwd@127.0.0.1:1235" }
+						{ url: "nats://user1:pwd@127.0.0.1:1234" }
+					]
+				}
+			`,
+			err:       fmt.Errorf("duplicate remote"),
+			errorLine: 7,
+			errorPos:  9,
+		},
 		{
 			name:       "show warnings on empty configs without values",
 			config:     ``,

server/gateway.go

@@ -1156,9 +1156,7 @@ func (c *client) processGatewayInfo(info *Info) {
 				// defensive code above that if we did not register this connection
 				// because we already have an outbound for this name, then
 				// close this connection (and make sure it does not try to reconnect)
-				c.mu.Lock()
-				c.flags.set(noReconnect)
-				c.mu.Unlock()
+				c.setNoReconnect()
 				c.closeConnection(WrongGateway)
 				return
 			}

server/jetstream_cluster_2_test.go

@@ -6141,8 +6141,11 @@ func TestJetStreamClusterLeafNodeSPOFMigrateLeadersWithMigrateDelay(t *testing.T
 	// Now make sure once leafnode is back we can have leaders on this server.
 	cl.reEnableLeafnodes()
 	checkLeafNodeConnectedCount(t, cl, 2)
-	for _, ln := range cl.leafRemoteCfgs {
-		require_True(t, ln.jsMigrateTimer == nil)
+	for ln := range cl.leafRemoteCfgs {
+		ln.RLock()
+		ok := ln.jsMigrateTimer == nil
+		ln.RUnlock()
+		require_True(t, ok)
 	}
 
 	// Make sure we can migrate back to this server now that we are connected.

server/jetstream_helpers_test.go

@@ -1176,7 +1176,7 @@ func (s *Server) reEnableLeafnodes() {
 // Helper to set the remote migrate feature.
 func (s *Server) setJetStreamMigrateOnRemoteLeaf() {
 	s.mu.Lock()
-	for _, cfg := range s.leafRemoteCfgs {
+	for cfg := range s.leafRemoteCfgs {
 		cfg.JetStreamClusterMigrate = true
 	}
 	s.mu.Unlock()
@@ -1185,7 +1185,7 @@ func (s *Server) setJetStreamMigrateOnRemoteLeaf() {
 // Helper to set the remote migrate feature.
 func (s *Server) setJetStreamMigrateOnRemoteLeafWithDelay(delay time.Duration) {
 	s.mu.Lock()
-	for _, cfg := range s.leafRemoteCfgs {
+	for cfg := range s.leafRemoteCfgs {
 		cfg.JetStreamClusterMigrate = true
 		cfg.JetStreamClusterMigrateDelay = delay
 	}

server/jetstream_leafnode_test.go

@@ -1279,7 +1279,10 @@ func TestJetStreamLeafNodeJSClusterMigrateRecovery(t *testing.T) {
 	// supposed to have and then take them down.
 	remotes := map[*Server]int{}
 	for _, s := range lnc.servers {
-		remotes[s] += len(s.leafRemoteCfgs)
+		s.mu.RLock()
+		count := len(s.leafRemoteCfgs)
+		s.mu.RUnlock()
+		remotes[s] += count
 		s.closeAndDisableLeafnodes()
 		checkLeafNodeConnectedCount(t, s, 0)
 	}
@@ -1362,7 +1365,10 @@ func TestJetStreamLeafNodeJSClusterMigrateRecoveryWithDelay(t *testing.T) {
 	// supposed to have and then take them down.
 	remotes := map[*Server]int{}
 	for _, s := range lnc.servers {
-		remotes[s] += len(s.leafRemoteCfgs)
+		s.mu.RLock()
+		count := len(s.leafRemoteCfgs)
+		s.mu.RUnlock()
+		remotes[s] += count
 		s.closeAndDisableLeafnodes()
 		checkLeafNodeConnectedCount(t, s, 0)
 	}
@@ -1418,8 +1424,11 @@ func TestJetStreamLeafNodeJSClusterMigrateRecoveryWithDelay(t *testing.T) {
 
 	// Make sure all delay timers in remotes are disabled
 	for _, s := range lnc.servers {
-		for _, r := range s.leafRemoteCfgs {
-			require_True(t, r.jsMigrateTimer == nil)
+		for r := range s.leafRemoteCfgs {
+			r.RLock()
+			ok := r.jsMigrateTimer == nil
+			r.RUnlock()
+			require_True(t, ok)
 		}
 	}