harden the UDN isolation tests to account for network blips

UDN tests are known to be heavy and these tests will fail on the
first dropped connection which seems to happen randomly, albeit
rarely.

test: allow single timeout in UDN KAPI reachability check

Hardens e2e test against transient networking blips while
preserving strict failure detection for real regressions.

Signed-off-by: Jamo Luhrsen <jluhrsen@gmail.com>

Author: jluhrsen

test/extended/networking/network_segmentation.go

@@ -392,42 +392,75 @@ var _ = Describe("[sig-network][OCPFeatureGate:NetworkSegmentation][Feature:User
 						}
 
 						By("asserting UDN pod can reach the kapi service in the default network")
-						// Use the service name to get test the DNS access
-						Consistently(func() bool {
-							_, err := e2ekubectl.RunKubectl(
+						// This is a positive reachability check that verifies the UDN pod can reach KAPI
+						// service via DNS. It tolerates one isolated curl timeout (observed on Azure) but
+						// still fails on consecutive failures or sustained connectivity issues that would
+						// indicate a UDN route/DNS/connectivity regression.
+						const (
+							requiredSuccesses = 3
+							maxTimeouts       = 1
+							kapiProbeWindow   = 30 * time.Second
+							kapiProbeInterval = 2 * time.Second
+						)
+
+						successCount := 0
+						timeoutCount := 0
+						consecutiveFailures := 0
+						deadline := time.Now().Add(kapiProbeWindow)
+
+						for time.Now().Before(deadline) && successCount < requiredSuccesses {
+							stdout, err := e2ekubectl.RunKubectl(
 								udnPodConfig.namespace,
 								"exec",
 								udnPodConfig.name,
 								"--",
 								"curl",
+								"--silent",
+								"--show-error",
+								"--fail",
 								"--connect-timeout",
-								// FIXME: We have seen in OCP CI that it can take two seconds or maybe more
-								// for a single curl to succeed. Example:
-								//     STEP: asserting UDN pod can reach the kapi service in the default network @ 01/20/25 00:38:42.32
-								// I0120 00:38:42.320808 70120 builder.go:121] Running '/usr/bin/kubectl
-								// --server=https://api.ci-op-bkg2qwwq-4edbf.XXXXXXXXXXXXXXXXXXXXXX:6443 --kubeconfig=/tmp/kubeconfig-1734723086
-								// --namespace=e2e-test-network-segmentation-e2e-kzdw7 exec udn-pod -- curl --connect-timeout 2 --insecure https://kubernetes.default/healthz'
-								// I0120 00:38:44.108334 70120 builder.go:146] stderr: "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n                                 Dload  Upload   Total   Spent    Left  Speed\n\r  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0\r100     2  100     2    0     0      9      0 --:--:-- --:--:-- --:--:--     9\r100     2  100     2    0     0      9      0 --:--:-- --:--:-- --:--:--     9\n"
-								// I0120 00:38:44.108415 70120 builder.go:147] stdout: "ok" --> 2 seconds later
-								// I0120 00:38:45.109237 70120 builder.go:121] Running '/usr/bin/kubectl
-								// --server=https://api.ci-op-bkg2qwwq-4edbf.XXXXXXXXXXXXXXXXXXXXXX:6443 --kubeconfig=/tmp/kubeconfig-1734723086
-								// --namespace=e2e-test-network-segmentation-e2e-kzdw7 exec udn-pod -- curl --connect-timeout 2 --insecure https://kubernetes.default/healthz'
-								// I0120 00:38:48.460089 70120 builder.go:135] rc: 28
-								// around the same time we have observed OVS issues like:
-								// Jan 20 00:38:45.329999 ci-op-bkg2qwwq-4edbf-xv8kb-worker-b-flqxd ovs-vswitchd[1094]: ovs|03661|timeval|WARN|context switches: 0 voluntary, 695 involuntary
-								// Jan 20 00:38:45.329967 ci-op-bkg2qwwq-4edbf-xv8kb-worker-b-flqxd ovs-vswitchd[1094]: ovs|03660|timeval|WARN|Unreasonably long 1730ms poll interval (32ms user, 903ms system)
-								// which might need more investigation. Bumping the timeout to 5seconds can help with this
-								// but we need to figure out what exactly is causing random timeouts in CI when trying to reach kapi-server
-								// sometimes we have also seen more than 2seconds being taken for the timeout which also needs to be investigated:
-								// I0118 13:35:50.419638 87083 builder.go:121] Running '/usr/bin/kubectl
-								// --server=https://api.ostest.test.metalkube.org:6443 --kubeconfig=/tmp/secret/kubeconfig
-								// --namespace=e2e-test-network-segmentation-e2e-d4fzk exec udn-pod -- curl --connect-timeout 2 --insecure https://kubernetes.default/healthz'
-								// I0118 13:35:54.093268 87083 builder.go:135] rc: 28 --> takes close to 4seconds?
 								"5",
+								"--max-time",
+								"10",
 								"--insecure",
-								"https://kubernetes.default/healthz")
-							return err == nil
-						}, 15*time.Second, 3*time.Second).Should(BeTrue())
+								"https://kubernetes.default/healthz",
+							)
+
+							if err == nil {
+								Expect(strings.TrimSpace(stdout)).To(Equal("ok"),
+									"unexpected response from kapi healthz")
+								successCount++
+								consecutiveFailures = 0
+								framework.Logf("UDN pod reached kapi healthz: success=%d/%d timeoutCount=%d",
+									successCount, requiredSuccesses, timeoutCount)
+								if successCount < requiredSuccesses {
+									time.Sleep(kapiProbeInterval)
+								}
+								continue
+							}
+
+							if isCurlExitCode28(err) {
+								timeoutCount++
+								consecutiveFailures++
+								framework.Logf("UDN pod kapi healthz curl timeout: timeoutCount=%d err=%v",
+									timeoutCount, err)
+
+								Expect(timeoutCount).To(BeNumerically("<=", maxTimeouts),
+									"only one transient timeout is allowed")
+								Expect(consecutiveFailures).To(BeNumerically("<=", 1),
+									"consecutive KAPI failures indicate sustained connectivity failure")
+
+								time.Sleep(kapiProbeInterval)
+								continue
+							}
+
+							Expect(err).NotTo(HaveOccurred(),
+								"non-timeout error reaching kapi healthz, stdout=%q", stdout)
+						}
+
+						Expect(successCount).To(BeNumerically(">=", requiredSuccesses),
+							"UDN pod did not reach kapi healthz enough times within %s; timeoutCount=%d",
+							kapiProbeWindow, timeoutCount)
 
 						By("asserting UDN pod can't reach default services via default network interface")
 						// route setup is already done, get kapi IPs
@@ -1285,6 +1318,11 @@ var _ = Describe("[sig-network][OCPFeatureGate:NetworkSegmentation][Feature:User
 	})
 })
 
+// isCurlExitCode28 checks if the error from RunKubectl indicates a curl timeout (exit code 28).
+func isCurlExitCode28(err error) bool {
+	return err != nil && strings.Contains(err.Error(), "rc: 28")
+}
+
 // randomNetworkMetaName return pseudo random name for network related objects (NAD,UDN,CUDN).
 // CUDN is cluster-scoped object, in case tests running in parallel, having random names avoids
 // conflicting with other tests.