fix(e2e): wait for flow cleanup before checking service after deletion in metallb test (#6359)

The metallb e2e test "should support metallb and underlay combine" fails
intermittently with curl exit code 56 (connection reset) when checking
the second service's reachability immediately after deleting the first
service. This happens because the OVN LoadBalancer data plane update and
the daemon's syncFlows (ovs-ofctl replace-flows) have not yet completed,
causing a brief disruption to the remaining service's data path.

Add a waitUnderlayServiceFlowCleaned helper that polls all nodes until
the deleted service's OpenFlow rules are fully removed from the external
bridge, ensuring the data plane has stabilized before checking the
second service.

Signed-off-by: Mengxin Liu <liumengxinfly@gmail.com>
Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>

Author: oilbeater

test/e2e/metallb/e2e_test.go

@@ -429,6 +429,11 @@ var _ = framework.Describe("[group:metallb]", func() {
 		ginkgo.By("Deleting the first service")
 		serviceClient.DeleteSync(serviceName)
 
+		ginkgo.By("Waiting for first service's underlay OpenFlow rules to be cleaned up")
+		for _, ingress := range service.Status.LoadBalancer.Ingress {
+			waitUnderlayServiceFlowCleaned(nodeNames, providerNetworkName, ingress.IP, curlListenPort, 15*time.Second)
+		}
+
 		ginkgo.By("Checking the second service is still reachable after first service deletion")
 		for i, ingress := range service2.Status.LoadBalancer.Ingress {
 			lbsvcIP2 := ingress.IP
@@ -614,3 +619,21 @@ func waitUnderlayServiceFlow(nodeName, providerNetworkName, serviceIP string, se
 
 	return flowFound
 }
+
+func waitUnderlayServiceFlowCleaned(nodeNames []string, providerNetworkName, serviceIP string, servicePort int32, timeout time.Duration) {
+	ginkgo.GinkgoHelper()
+
+	bridgeName := util.ExternalBridgeName(providerNetworkName)
+	matchPort := fmt.Sprintf("tp_dst=%d", servicePort)
+
+	framework.WaitUntil(1*time.Second, timeout, func(_ context.Context) (bool, error) {
+		for _, nodeName := range nodeNames {
+			cmd := fmt.Sprintf("kubectl ko ofctl %s dump-flows %s | grep -w %s | grep -w %s",
+				nodeName, bridgeName, serviceIP, matchPort)
+			if _, err := exec.Command("bash", "-c", cmd).CombinedOutput(); err == nil {
+				return false, nil // flow still exists on this node
+			}
+		}
+		return true, nil // flow cleaned from all nodes
+	}, fmt.Sprintf("underlay service flow for %s should be cleaned up", serviceIP))
+}