system-tests: fix IPv6 DAD check order and error propagation (#1294)

Fix two related bugs in IPv6 Duplicate Address Detection (DAD) handling:

1. Check order bug: Functions checked for "tentative" state before
   "dadfailed" state, preventing detection of permanent DAD failures
   when both flags were present (e.g., "dadfailed tentative").

2. Error propagation bug: DAD failure errors were not propagated
   correctly, causing tests to retry for 3 minutes instead of failing
   immediately with clear error messages.

Changes:
- checkIPv6AddressState() and checkPodIPv6Ready(): Swap check order to
  detect "dadfailed" before "tentative" (permanent before transient)

- whereabouts-statefulset.go (getPodWhereaboutsIPs):
  * Change Eventually from func() bool to func() error pattern
  * Use StopTrying() for DAD failures to abort Gomega Eventually immediately
  * Return regular error for transient failures (exec/scanner errors) to retry

- sriov-pod-level-bond.go (inspectPodLevelBondedInterfaceConfig):
  * Return error for DAD failures to abort k8s PollUntilContextTimeout immediately
  * Return false, nil for transient failures (scanner errors) to continue polling

Impact: Tests with DAD failures now fail immediately (subsecond) with specific
"IPv6 DAD failed" errors instead of timing out after 3 minutes with generic
error messages.

Co-authored-by: Claude Sonnet 4.5 <noreply@anthropic.com>

Author: yprokule

tests/system-tests/rdscore/internal/rdscorecommon/sriov-pod-level-bond.go

@@ -667,20 +667,20 @@ func checkIPv6AddressState(output, ipv6Addr string) (bool, error) {
 
 		// Look for lines containing the IPv6 address
 		if strings.Contains(line, ipv6Addr) && strings.Contains(line, "inet6") {
-			// Check if the address is in tentative state (DAD in progress)
-			if strings.Contains(line, "tentative") {
-				klog.V(100).Infof("IPv6 address %s is in tentative state (DAD in progress): %s",
+			// Check if DAD failed (permanent failure - must check first)
+			if strings.Contains(line, "dadfailed") {
+				klog.V(100).Infof("IPv6 address %s DAD failed: %s",
 					ipv6Addr, strings.TrimSpace(line))
 
-				return false, nil
+				return false, fmt.Errorf("IPv6 DAD failed for address %s", ipv6Addr)
 			}
 
-			// Check if DAD failed
-			if strings.Contains(line, "dadfailed") {
-				klog.V(100).Infof("IPv6 address %s DAD failed: %s",
+			// Check if the address is in tentative state (DAD in progress)
+			if strings.Contains(line, "tentative") {
+				klog.V(100).Infof("IPv6 address %s is in tentative state (DAD in progress): %s",
 					ipv6Addr, strings.TrimSpace(line))
 
-				return false, fmt.Errorf("IPv6 DAD failed for address %s", ipv6Addr)
+				return false, nil
 			}
 
 			// Address found and not tentative - ready to use
@@ -704,7 +704,7 @@ func checkIPv6AddressState(output, ipv6Addr string) (bool, error) {
 	return false, nil
 }
 
-//nolint:funlen
+//nolint:funlen,gocognit
 func inspectPodLevelBondedInterfaceConfig(podObj *pod.Builder, ipv4Addr, ipv6Addr string) (bool, error) {
 	klog.V(100).Infof("Verify pod-level bonded interface configuration for pod %q in namespace %q",
 		podObj.Definition.Name, podObj.Definition.Namespace)
@@ -782,10 +782,19 @@ func inspectPodLevelBondedInterfaceConfig(podObj *pod.Builder, ipv4Addr, ipv6Add
 				// Check that IPv6 address is not in tentative state (DAD must be complete)
 				ipv6Ready, err := checkIPv6AddressState(output, ipv6Addr)
 				if err != nil {
-					klog.V(100).Infof("IPv6 address %s DAD failed for pod %s in namespace %s: %v",
+					// Check if this is a permanent DAD failure
+					if strings.Contains(err.Error(), "IPv6 DAD failed") {
+						klog.V(100).Infof("IPv6 address %s DAD failed for pod %s in namespace %s: %v",
+							ipv6Addr, podObj.Definition.Name, podObj.Definition.Namespace, err)
+
+						return false, err
+					}
+
+					// For other errors (scanner errors), continue retrying
+					klog.V(100).Infof("Temporary error checking IPv6 address %s for pod %s in namespace %s: %v, will retry",
 						ipv6Addr, podObj.Definition.Name, podObj.Definition.Namespace, err)
 
-					return false, err
+					return false, nil
 				}
 
 				if !ipv6Ready {

tests/system-tests/rdscore/internal/rdscorecommon/whereabouts-statefulset.go

@@ -373,6 +373,14 @@ func checkPodIPv6Ready(podObj *pod.Builder, interfaceName, ipv6Addr string) (boo
 
 		// Look for lines containing the IPv6 address
 		if strings.Contains(line, ipv6Addr) && strings.Contains(line, "inet6") {
+			// Check if DAD failed (permanent failure - must check first)
+			if strings.Contains(line, "dadfailed") {
+				klog.V(rdscoreparams.RDSCoreLogLevel).Infof("IPv6 address %s DAD failed for pod %q: %s",
+					ipv6Addr, podObj.Object.Name, strings.TrimSpace(line))
+
+				return false, fmt.Errorf("IPv6 DAD failed for address %s in pod %s", ipv6Addr, podObj.Object.Name)
+			}
+
 			// Check if the address is in tentative state (DAD in progress)
 			if strings.Contains(line, "tentative") {
 				klog.V(rdscoreparams.RDSCoreLogLevel).Infof(
@@ -382,14 +390,6 @@ func checkPodIPv6Ready(podObj *pod.Builder, interfaceName, ipv6Addr string) (boo
 				return false, nil
 			}
 
-			// Check if DAD failed
-			if strings.Contains(line, "dadfailed") {
-				klog.V(rdscoreparams.RDSCoreLogLevel).Infof("IPv6 address %s DAD failed for pod %q: %s",
-					ipv6Addr, podObj.Object.Name, strings.TrimSpace(line))
-
-				return false, fmt.Errorf("IPv6 DAD failed for address %s in pod %s", ipv6Addr, podObj.Object.Name)
-			}
-
 			// Address found and not tentative - ready to use
 			klog.V(rdscoreparams.RDSCoreLogLevel).Infof("IPv6 address %s is ready (not tentative) for pod %q",
 				ipv6Addr, podObj.Object.Name)
@@ -415,6 +415,8 @@ func checkPodIPv6Ready(podObj *pod.Builder, interfaceName, ipv6Addr string) (boo
 }
 
 // getPodWhereaboutsIPs gets the IP addresses for the given pod.
+//
+//nolint:gocognit
 func getPodWhereaboutsIPs(activePods []*pod.Builder, interfaceName string) map[string][]NetworkInterface {
 	podWhereaboutsIPs := make(map[string][]NetworkInterface)
 
@@ -425,7 +427,7 @@ func getPodWhereaboutsIPs(activePods []*pod.Builder, interfaceName string) map[s
 	for _, _pod := range activePods {
 		var networkInterface []NetworkInterface
 
-		Eventually(func() bool {
+		Eventually(func() error {
 			klog.V(rdscoreparams.RDSCoreLogLevel).Infof("Executing command %q within pod %q in %q namespace",
 				cmdGetIPAddr, _pod.Object.Name, _pod.Object.Namespace)
 
@@ -434,14 +436,14 @@ func getPodWhereaboutsIPs(activePods []*pod.Builder, interfaceName string) map[s
 				klog.V(rdscoreparams.RDSCoreLogLevel).Infof("Failed to execute command within pod %q in %q namespace: %s",
 					_pod.Object.Name, _pod.Object.Namespace, err)
 
-				return false
+				return fmt.Errorf("failed to execute command in pod: %w", err)
 			}
 
 			if addrBuffInfo.Len() == 0 {
 				klog.V(rdscoreparams.RDSCoreLogLevel).Infof("Empty output from command within pod %q in %q namespace",
 					_pod.Object.Name, _pod.Object.Namespace)
 
-				return false
+				return fmt.Errorf("empty output from command")
 			}
 
 			klog.V(rdscoreparams.RDSCoreLogLevel).Infof("Unmarshalling IP addresses")
@@ -451,7 +453,7 @@ func getPodWhereaboutsIPs(activePods []*pod.Builder, interfaceName string) map[s
 				klog.V(rdscoreparams.RDSCoreLogLevel).Infof("Failed to unmarshal IP addresses for pod %q in %q namespace: %s",
 					_pod.Object.Name, _pod.Object.Namespace, err)
 
-				return false
+				return fmt.Errorf("failed to unmarshal IP addresses: %w", err)
 			}
 
 			klog.V(rdscoreparams.RDSCoreLogLevel).Infof("IP addresses: %+v", networkInterface)
@@ -464,19 +466,31 @@ func getPodWhereaboutsIPs(activePods []*pod.Builder, interfaceName string) map[s
 
 					ipv6Ready, err := checkPodIPv6Ready(_pod, interfaceName, addr.Local)
 					if err != nil {
+						// Check if this is a permanent DAD failure
+						if strings.Contains(err.Error(), "IPv6 DAD failed") {
+							klog.V(rdscoreparams.RDSCoreLogLevel).Infof(
+								"IPv6 address %s DAD failed for pod %q in %q namespace: %v",
+								addr.Local, _pod.Object.Name, _pod.Object.Namespace, err)
+
+							return StopTrying(fmt.Sprintf(
+								"IPv6 DAD failed for address %s in pod %s/%s",
+								addr.Local, _pod.Object.Namespace, _pod.Object.Name)).Wrap(err)
+						}
+
+						// For other errors (exec failures, scanner errors), continue retrying
 						klog.V(rdscoreparams.RDSCoreLogLevel).Infof(
-							"IPv6 address %s DAD failed for pod %q in %q namespace: %v",
+							"Temporary error checking IPv6 address %s for pod %q in %q namespace: %v, will retry",
 							addr.Local, _pod.Object.Name, _pod.Object.Namespace, err)
 
-						return false
+						return fmt.Errorf("failed to check IPv6 address: %w", err)
 					}
 
 					if !ipv6Ready {
 						klog.V(rdscoreparams.RDSCoreLogLevel).Infof(
 							"IPv6 address %s not ready yet (tentative state) for pod %q in %q namespace, retrying...",
 							addr.Local, _pod.Object.Name, _pod.Object.Namespace)
 
-						return false
+						return fmt.Errorf("IPv6 address not ready (tentative state)")
 					}
 
 					klog.V(rdscoreparams.RDSCoreLogLevel).Infof(
@@ -487,8 +501,8 @@ func getPodWhereaboutsIPs(activePods []*pod.Builder, interfaceName string) map[s
 
 			podWhereaboutsIPs[_pod.Object.Name] = networkInterface
 
-			return true
-		}).WithContext(ctx).WithPolling(15*time.Second).WithTimeout(3*time.Minute).Should(BeTrue(),
+			return nil
+		}).WithContext(ctx).WithPolling(15*time.Second).WithTimeout(3*time.Minute).Should(Succeed(),
 			"Failed to get IP addresses for pod %q in %q namespace", _pod.Object.Name, _pod.Object.Namespace)
 	}