fix(e2e): reuse OVSDB client in address set tests to avoid HA deadlock (#6368)

Cache the OVN NB client across WaitForAddressSet* calls instead of
creating a new connection each time. In HA environments, repeated
connect/monitor cycles can trigger a deadlock in libovsdb when a leader
change occurs during the initial Monitor RPC setup: the Monitor goroutine
holds rpcMutex.RLock while the leader-change handler blocks on
rpcMutex.Lock, and Go's write-preferring RWMutex then also blocks the
inactivity probe's RLock, leaving no recovery path.

Also tolerate transient "not connected" errors during the poll loop
instead of failing immediately, giving libovsdb's automatic reconnection
time to recover after a leader failover.

Signed-off-by: Mengxin Liu <liumengxinfly@gmail.com>
Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>

Author: oilbeater

test/e2e/framework/ovn_address_set.go

@@ -37,9 +37,13 @@ const (
 )
 
 var (
-	ovnnbAddrOnce sync.Once
-	ovnnbAddrErr  error
-	ovnnbAddr     string
+	ovnnbAddrOnce    sync.Once
+	ovnnbAddrErr     error
+	ovnnbAddr        string
+	ovnnbClientOnce  sync.Once
+	ovnnbClientErr   error
+	ovnnbCachedNbCli *ovs.OVNNbClient
+	ovnnbCachedModel map[string]model.Model
 )
 
 func validateModelStructure(model model.Model, table string, expectedFields map[string]reflect.Type) {
@@ -61,8 +65,6 @@ func WaitForAddressSetCondition(condition func(rows any) (bool, error)) {
 	ginkgo.GinkgoHelper()
 
 	client, models := getOVNNbClient(ovnnb.AddressSetTable)
-	defer client.Close()
-
 	model := models[ovnnb.AddressSetTable]
 	validateModelStructure(model, ovnnb.AddressSetTable, map[string]reflect.Type{
 		"Name":        reflect.TypeFor[string](),
@@ -76,7 +78,10 @@ func WaitForAddressSetCondition(condition func(rows any) (bool, error)) {
 
 		result := reflect.New(reflect.SliceOf(reflect.TypeOf(model).Elem())).Interface()
 		if err := client.List(ctx, result); err != nil {
-			return false, err
+			// Transient errors (e.g. "not connected" during leader failover)
+			// should be retried rather than immediately failing the poll.
+			Logf("Failed to list address sets (will retry): %v", err)
+			return false, nil
 		}
 
 		return condition(result)
@@ -176,16 +181,18 @@ func getOVNNbClient(tables ...string) (*ovs.OVNNbClient, map[string]model.Model)
 	})
 	ExpectNoError(ovnnbAddrErr)
 
-	client, models, err := ovs.NewDynamicOvnNbClient(
-		ovnnbAddr,
-		ovnNbTimeoutSeconds,
-		ovsdbConnTimeout,
-		ovsdbInactivityTimeout,
-		tables...,
-	)
-	ExpectNoError(err)
+	ovnnbClientOnce.Do(func() {
+		ovnnbCachedNbCli, ovnnbCachedModel, ovnnbClientErr = ovs.NewDynamicOvnNbClient(
+			ovnnbAddr,
+			ovnNbTimeoutSeconds,
+			ovsdbConnTimeout,
+			ovsdbInactivityTimeout,
+			tables...,
+		)
+	})
+	ExpectNoError(ovnnbClientErr)
 
-	return client, models
+	return ovnnbCachedNbCli, ovnnbCachedModel
 }
 
 func resolveOVNNbConnection() (string, error) {