diff --git a/tests/system-tests/rdscore/internal/rdscorecommon/egress-ip.go b/tests/system-tests/rdscore/internal/rdscorecommon/egress-ip.go index 68d0ef6fe..aab35ae9e 100644 --- a/tests/system-tests/rdscore/internal/rdscorecommon/egress-ip.go +++ b/tests/system-tests/rdscore/internal/rdscorecommon/egress-ip.go @@ -1020,23 +1020,48 @@ func VerifyEgressIPForPodWithWrongLabel() { fmt.Sprintf("Failed to retrieve configured EgressIP addresses list from the egressIP %s: %v", RDSCoreConfig.EgressIPName, err)) - podObjectsPodObjects, err := pod.List(APIClient, RDSCoreConfig.EgressIPNamespaceOne, egressIPPodSelector) - Expect(err).ToNot(HaveOccurred(), - fmt.Sprintf("Failed to retrieve pods list from namespace %s with label %s: %v", - RDSCoreConfig.EgressIPNamespaceOne, RDSCoreConfig.EgressIPPodLabel, err)) + type ipFamilyEntry struct { + name string + isIPv6 bool + } - err = sendTrafficCheckIP(podObjectsPodObjects, false, expectedIPs) - Expect(err).ToNot(HaveOccurred(), - fmt.Sprintf("Server response was note received: %v", err)) + var ipFamilies []ipFamilyEntry - podObjectsPodObjects, err = pod.List(APIClient, RDSCoreConfig.EgressIPNamespaceOne, nonEgressIPPodSelector) - Expect(err).ToNot(HaveOccurred(), - fmt.Sprintf("Failed to retrieve pods list from namespace %s with label %v: %v", - RDSCoreConfig.EgressIPNamespaceOne, nonEgressIPPodSelector, err)) + if RDSCoreConfig.EgressIPRemoteIPv4 != "" { + ipFamilies = append(ipFamilies, ipFamilyEntry{name: "IPv4", isIPv6: false}) + } + + if RDSCoreConfig.EgressIPRemoteIPv6 != "" { + ipFamilies = append(ipFamilies, ipFamilyEntry{name: "IPv6", isIPv6: true}) + } + + Expect(len(ipFamilies)).ToNot(Equal(0), + "Neither EgressIPRemoteIPv4 nor EgressIPRemoteIPv6 is configured") - err = sendTrafficCheckIP(podObjectsPodObjects, false, expectedIPs) - Expect(err).To(HaveOccurred(), - fmt.Sprintf("Server response was received with the not correct egressIP address: %v", err)) + for _, family := range ipFamilies { + By(fmt.Sprintf("Positive check: verifying EgressIP is used for correctly-labeled pods (%s)", family.name)) + + correctLabelPods, err := pod.List(APIClient, RDSCoreConfig.EgressIPNamespaceOne, egressIPPodSelector) + Expect(err).ToNot(HaveOccurred(), + fmt.Sprintf("Failed to retrieve pods list from namespace %s with label %s: %v", + RDSCoreConfig.EgressIPNamespaceOne, RDSCoreConfig.EgressIPPodLabel, err)) + + err = sendTrafficCheckIP(correctLabelPods, family.isIPv6, expectedIPs) + Expect(err).ToNot(HaveOccurred(), + fmt.Sprintf("%s: server response was not received from correctly-labeled pods: %v", family.name, err)) + + By(fmt.Sprintf("Negative check: verifying EgressIP is NOT used for wrong-labeled pods (%s)", family.name)) + + wrongLabelPods, err := pod.List(APIClient, RDSCoreConfig.EgressIPNamespaceOne, nonEgressIPPodSelector) + Expect(err).ToNot(HaveOccurred(), + fmt.Sprintf("Failed to retrieve pods list from namespace %s with label %v: %v", + RDSCoreConfig.EgressIPNamespaceOne, nonEgressIPPodSelector, err)) + + err = sendTrafficCheckIP(wrongLabelPods, family.isIPv6, expectedIPs) + Expect(err).To(HaveOccurred(), + fmt.Sprintf("%s: server response was received with egressIP from wrong-labeled pods: %v", + family.name, err)) + } } // VerifyEgressIPForNamespaceWithWrongLabel verifies egress traffic applies only for the pods @@ -1074,9 +1099,32 @@ func VerifyEgressIPForNamespaceWithWrongLabel() { fmt.Sprintf("Failed to retrieve pods list from namespace %s with label %s: %v", nonEgressIPNamespace, RDSCoreConfig.EgressIPPodLabel, err)) - err = sendTrafficCheckIP(podObjects, false, expectedIPs) - Expect(err).To(HaveOccurred(), - fmt.Sprintf("Server response was received with the not correct egressIP address: %v", err)) + type ipFamilyEntry struct { + name string + isIPv6 bool + } + + var ipFamilies []ipFamilyEntry + + if RDSCoreConfig.EgressIPRemoteIPv4 != "" { + ipFamilies = append(ipFamilies, ipFamilyEntry{name: "IPv4", isIPv6: false}) + } + + if RDSCoreConfig.EgressIPRemoteIPv6 != "" { + ipFamilies = append(ipFamilies, ipFamilyEntry{name: "IPv6", isIPv6: true}) + } + + Expect(len(ipFamilies)).ToNot(Equal(0), + "Neither EgressIPRemoteIPv4 nor EgressIPRemoteIPv6 is configured") + + for _, family := range ipFamilies { + By(fmt.Sprintf("Verifying EgressIP is NOT used for pods in wrong namespace (%s)", family.name)) + + err = sendTrafficCheckIP(podObjects, family.isIPv6, expectedIPs) + Expect(err).To(HaveOccurred(), + fmt.Sprintf("%s: server response was received with egressIP from wrong namespace: %v", + family.name, err)) + } } // VerifyEgressIPOneNamespaceThreeNodesBalancedEIPTrafficIPv4 verifies egress traffic works with egressIP diff --git a/tests/system-tests/rdscore/internal/rdscorecommon/egress-service.go b/tests/system-tests/rdscore/internal/rdscorecommon/egress-service.go index b7e4f28fc..d4d4f290c 100644 --- a/tests/system-tests/rdscore/internal/rdscorecommon/egress-service.go +++ b/tests/system-tests/rdscore/internal/rdscorecommon/egress-service.go @@ -382,12 +382,35 @@ func VerifyEgressServiceETPClusterWrapper( svcBuilder = svcBuilder.WithAnnotation(map[string]string{ "metallb.universe.tf/address-pool": ipAddrPoolName}) - By("Setting ipFamilyPolicy to 'RequireDualStack'") + hasIPv4 := remoteTargetIP != "" + hasIPv6 := remoteTargetIPv6 != "" - klog.V(rdscoreparams.RDSCoreLogLevel).Infof("Setting ipFamilyPolicy to 'RequireDualStack'") + Expect(hasIPv4 || hasIPv6).To(BeTrue(), + "At least one remote target IP (IPv4 or IPv6) must be configured") - svcBuilder = svcBuilder.WithIPFamily([]corev1.IPFamily{"IPv4", "IPv6"}, - corev1.IPFamilyPolicyRequireDualStack) + switch { + case hasIPv4 && hasIPv6: + By("Setting ipFamilyPolicy to 'RequireDualStack'") + + klog.V(rdscoreparams.RDSCoreLogLevel).Infof("Dual-stack: setting ipFamilyPolicy to RequireDualStack") + + svcBuilder = svcBuilder.WithIPFamily([]corev1.IPFamily{corev1.IPv4Protocol, corev1.IPv6Protocol}, + corev1.IPFamilyPolicyRequireDualStack) + case hasIPv6: + By("Setting ipFamilyPolicy to 'SingleStack' (IPv6)") + + klog.V(rdscoreparams.RDSCoreLogLevel).Infof("IPv6 only: setting ipFamilyPolicy to SingleStack") + + svcBuilder = svcBuilder.WithIPFamily([]corev1.IPFamily{corev1.IPv6Protocol}, + corev1.IPFamilyPolicySingleStack) + default: + By("Setting ipFamilyPolicy to 'SingleStack' (IPv4)") + + klog.V(rdscoreparams.RDSCoreLogLevel).Infof("IPv4 only: setting ipFamilyPolicy to SingleStack") + + svcBuilder = svcBuilder.WithIPFamily([]corev1.IPFamily{corev1.IPv4Protocol}, + corev1.IPFamilyPolicySingleStack) + } By("Creating a service") @@ -675,10 +698,32 @@ func VerifyEgressServiceWithLocalETPWrapper( svcBuilder = svcBuilder.WithAnnotation(map[string]string{ "metallb.universe.tf/address-pool": ipAddrPoolName}) - By("Setting ipFamilyPolicy to 'RequireDualStack'") + hasIPv4 := remoteTargetIP != "" + hasIPv6 := remoteTargetIPv6 != "" - svcBuilder = svcBuilder.WithIPFamily([]corev1.IPFamily{"IPv4", "IPv6"}, - corev1.IPFamilyPolicyRequireDualStack) + switch { + case hasIPv4 && hasIPv6: + By("Setting ipFamilyPolicy to 'RequireDualStack'") + + klog.V(rdscoreparams.RDSCoreLogLevel).Infof("Dual-stack: setting ipFamilyPolicy to RequireDualStack") + + svcBuilder = svcBuilder.WithIPFamily([]corev1.IPFamily{corev1.IPv4Protocol, corev1.IPv6Protocol}, + corev1.IPFamilyPolicyRequireDualStack) + case hasIPv6: + By("Setting ipFamilyPolicy to 'SingleStack' (IPv6)") + + klog.V(rdscoreparams.RDSCoreLogLevel).Infof("IPv6 only: setting ipFamilyPolicy to SingleStack") + + svcBuilder = svcBuilder.WithIPFamily([]corev1.IPFamily{corev1.IPv6Protocol}, + corev1.IPFamilyPolicySingleStack) + default: + By("Setting ipFamilyPolicy to 'SingleStack' (IPv4)") + + klog.V(rdscoreparams.RDSCoreLogLevel).Infof("IPv4 only: setting ipFamilyPolicy to SingleStack") + + svcBuilder = svcBuilder.WithIPFamily([]corev1.IPFamily{corev1.IPv4Protocol}, + corev1.IPFamilyPolicySingleStack) + } By("Creating a service") klog.V(rdscoreparams.RDSCoreLogLevel).Infof("Creating Service object") @@ -953,73 +998,117 @@ func verifySourceIP(svcName, svcNS, podLabels string, cmdToRun []string, useIPv6 // VerifyEgressServiceConnectivityETPCluster verifies source IP address when external traffic policy // is set to Cluster. func VerifyEgressServiceConnectivityETPCluster() { - cmdToRun := []string{"/bin/bash", "-c", - fmt.Sprintf("curl --connect-timeout 3 -Ls http://%s:%s/clientip", - RDSCoreConfig.EgressServiceRemoteIP, RDSCoreConfig.EgressServiceRemotePort)} + Expect(RDSCoreConfig.EgressServiceRemoteIP != "" || RDSCoreConfig.EgressServiceRemoteIPv6 != "").To(BeTrue(), + "Neither EgressServiceRemoteIP nor EgressServiceRemoteIPv6 is configured") + + if RDSCoreConfig.EgressServiceRemoteIP != "" { + By("Verifying EgressService ETP=Cluster connectivity (IPv4)") - verifySourceIP(egressSVC1Name, RDSCoreConfig.EgressServiceNS, egressSVC1Labels, cmdToRun, false, - RDSCoreConfig.EgressServiceNetworkExpectedIPs) + cmdToRun := []string{"/bin/bash", "-c", + fmt.Sprintf("curl --connect-timeout 3 -Ls http://%s:%s/clientip", + RDSCoreConfig.EgressServiceRemoteIP, RDSCoreConfig.EgressServiceRemotePort)} - cmdToRun = []string{"/bin/bash", "-c", - fmt.Sprintf("curl --connect-timeout 3 -Ls http://[%s]:%s/clientip", - RDSCoreConfig.EgressServiceRemoteIPv6, RDSCoreConfig.EgressServiceRemotePort)} + verifySourceIP(egressSVC1Name, RDSCoreConfig.EgressServiceNS, egressSVC1Labels, cmdToRun, false, + RDSCoreConfig.EgressServiceNetworkExpectedIPs) + } + + if RDSCoreConfig.EgressServiceRemoteIPv6 != "" { + By("Verifying EgressService ETP=Cluster connectivity (IPv6)") + + cmdToRun := []string{"/bin/bash", "-c", + fmt.Sprintf("curl --connect-timeout 3 -Ls http://[%s]:%s/clientip", + RDSCoreConfig.EgressServiceRemoteIPv6, RDSCoreConfig.EgressServiceRemotePort)} - verifySourceIP(egressSVC1Name, RDSCoreConfig.EgressServiceNS, egressSVC1Labels, cmdToRun, true, - RDSCoreConfig.EgressServiceNetworkExpectedIPs) + verifySourceIP(egressSVC1Name, RDSCoreConfig.EgressServiceNS, egressSVC1Labels, cmdToRun, true, + RDSCoreConfig.EgressServiceNetworkExpectedIPs) + } } // VerifyEgressServiceConnectivityETPClusterSourceIPByNetwork verifies source IP address when external traffic policy // is set to Cluster. func VerifyEgressServiceConnectivityETPClusterSourceIPByNetwork() { - cmdToRun := []string{"/bin/bash", "-c", - fmt.Sprintf("curl --connect-timeout 3 -Ls http://%s:%s/clientip", - RDSCoreConfig.EgressServiceRemoteIP, RDSCoreConfig.EgressServiceRemotePort)} + Expect(RDSCoreConfig.EgressServiceRemoteIP != "" || RDSCoreConfig.EgressServiceRemoteIPv6 != "").To(BeTrue(), + "Neither EgressServiceRemoteIP nor EgressServiceRemoteIPv6 is configured") + + if RDSCoreConfig.EgressServiceRemoteIP != "" { + By("Verifying EgressService ETP=Cluster sourceIPBy=Network connectivity (IPv4)") - verifySourceIP(egressSVC3Name, RDSCoreConfig.EgressServiceNS, egressSVC3Labels, cmdToRun, false, - RDSCoreConfig.EgressServiceNetworkExpectedIPs) + cmdToRun := []string{"/bin/bash", "-c", + fmt.Sprintf("curl --connect-timeout 3 -Ls http://%s:%s/clientip", + RDSCoreConfig.EgressServiceRemoteIP, RDSCoreConfig.EgressServiceRemotePort)} - cmdToRun = []string{"/bin/bash", "-c", - fmt.Sprintf("curl --connect-timeout 3 -Ls http://[%s]:%s/clientip", - RDSCoreConfig.EgressServiceRemoteIPv6, RDSCoreConfig.EgressServiceRemotePort)} + verifySourceIP(egressSVC3Name, RDSCoreConfig.EgressServiceNS, egressSVC3Labels, cmdToRun, false, + RDSCoreConfig.EgressServiceNetworkExpectedIPs) + } + + if RDSCoreConfig.EgressServiceRemoteIPv6 != "" { + By("Verifying EgressService ETP=Cluster sourceIPBy=Network connectivity (IPv6)") + + cmdToRun := []string{"/bin/bash", "-c", + fmt.Sprintf("curl --connect-timeout 3 -Ls http://[%s]:%s/clientip", + RDSCoreConfig.EgressServiceRemoteIPv6, RDSCoreConfig.EgressServiceRemotePort)} - verifySourceIP(egressSVC3Name, RDSCoreConfig.EgressServiceNS, egressSVC3Labels, cmdToRun, true, - RDSCoreConfig.EgressServiceNetworkExpectedIPs) + verifySourceIP(egressSVC3Name, RDSCoreConfig.EgressServiceNS, egressSVC3Labels, cmdToRun, true, + RDSCoreConfig.EgressServiceNetworkExpectedIPs) + } } // VerifyEgressServiceConnectivityETPLocal verifies source IP address when external traffic policy // is set to Local. func VerifyEgressServiceConnectivityETPLocal() { - cmdToRun := []string{"/bin/bash", "-c", - fmt.Sprintf("curl --connect-timeout 3 -Ls http://%s:%s/clientip", - RDSCoreConfig.EgressServiceRemoteIP, RDSCoreConfig.EgressServiceRemotePort)} + Expect(RDSCoreConfig.EgressServiceRemoteIP != "" || RDSCoreConfig.EgressServiceRemoteIPv6 != "").To(BeTrue(), + "Neither EgressServiceRemoteIP nor EgressServiceRemoteIPv6 is configured") - verifySourceIP(egressSVC2Name, RDSCoreConfig.EgressServiceNS, egressSVC2Labels, cmdToRun, false, - RDSCoreConfig.EgressServiceNetworkExpectedIPs) + if RDSCoreConfig.EgressServiceRemoteIP != "" { + By("Verifying EgressService ETP=Local connectivity (IPv4)") - cmdToRun = []string{"/bin/bash", "-c", - fmt.Sprintf("curl --connect-timeout 3 -Ls http://[%s]:%s/clientip", - RDSCoreConfig.EgressServiceRemoteIPv6, RDSCoreConfig.EgressServiceRemotePort)} + cmdToRun := []string{"/bin/bash", "-c", + fmt.Sprintf("curl --connect-timeout 3 -Ls http://%s:%s/clientip", + RDSCoreConfig.EgressServiceRemoteIP, RDSCoreConfig.EgressServiceRemotePort)} - verifySourceIP(egressSVC2Name, RDSCoreConfig.EgressServiceNS, egressSVC2Labels, cmdToRun, true, - RDSCoreConfig.EgressServiceNetworkExpectedIPs) + verifySourceIP(egressSVC2Name, RDSCoreConfig.EgressServiceNS, egressSVC2Labels, cmdToRun, false, + RDSCoreConfig.EgressServiceNetworkExpectedIPs) + } + + if RDSCoreConfig.EgressServiceRemoteIPv6 != "" { + By("Verifying EgressService ETP=Local connectivity (IPv6)") + + cmdToRun := []string{"/bin/bash", "-c", + fmt.Sprintf("curl --connect-timeout 3 -Ls http://[%s]:%s/clientip", + RDSCoreConfig.EgressServiceRemoteIPv6, RDSCoreConfig.EgressServiceRemotePort)} + + verifySourceIP(egressSVC2Name, RDSCoreConfig.EgressServiceNS, egressSVC2Labels, cmdToRun, true, + RDSCoreConfig.EgressServiceNetworkExpectedIPs) + } } // VerifyEgressServiceConnectivityETPLocalSourceIPByNetwork verifies source IP address when external traffic policy // is set to Local and sourceIPBy=Network. func VerifyEgressServiceConnectivityETPLocalSourceIPByNetwork() { - cmdToRun := []string{"/bin/bash", "-c", - fmt.Sprintf("curl --connect-timeout 3 -Ls http://%s:%s/clientip", - RDSCoreConfig.EgressServiceRemoteIP, RDSCoreConfig.EgressServiceRemotePort)} + Expect(RDSCoreConfig.EgressServiceRemoteIP != "" || RDSCoreConfig.EgressServiceRemoteIPv6 != "").To(BeTrue(), + "Neither EgressServiceRemoteIP nor EgressServiceRemoteIPv6 is configured") - verifySourceIP(egressSVC4Name, RDSCoreConfig.EgressServiceNS, egressSVC4Labels, cmdToRun, false, - RDSCoreConfig.EgressServiceNetworkExpectedIPs) + if RDSCoreConfig.EgressServiceRemoteIP != "" { + By("Verifying EgressService ETP=Local sourceIPBy=Network connectivity (IPv4)") - cmdToRun = []string{"/bin/bash", "-c", - fmt.Sprintf("curl --connect-timeout 3 -Ls http://[%s]:%s/clientip", - RDSCoreConfig.EgressServiceRemoteIPv6, RDSCoreConfig.EgressServiceRemotePort)} + cmdToRun := []string{"/bin/bash", "-c", + fmt.Sprintf("curl --connect-timeout 3 -Ls http://%s:%s/clientip", + RDSCoreConfig.EgressServiceRemoteIP, RDSCoreConfig.EgressServiceRemotePort)} - verifySourceIP(egressSVC4Name, RDSCoreConfig.EgressServiceNS, egressSVC4Labels, cmdToRun, true, - RDSCoreConfig.EgressServiceNetworkExpectedIPs) + verifySourceIP(egressSVC4Name, RDSCoreConfig.EgressServiceNS, egressSVC4Labels, cmdToRun, false, + RDSCoreConfig.EgressServiceNetworkExpectedIPs) + } + + if RDSCoreConfig.EgressServiceRemoteIPv6 != "" { + By("Verifying EgressService ETP=Local sourceIPBy=Network connectivity (IPv6)") + + cmdToRun := []string{"/bin/bash", "-c", + fmt.Sprintf("curl --connect-timeout 3 -Ls http://[%s]:%s/clientip", + RDSCoreConfig.EgressServiceRemoteIPv6, RDSCoreConfig.EgressServiceRemotePort)} + + verifySourceIP(egressSVC4Name, RDSCoreConfig.EgressServiceNS, egressSVC4Labels, cmdToRun, true, + RDSCoreConfig.EgressServiceNetworkExpectedIPs) + } } // VerifyEgressServiceETPLocalIngressConnectivity verifies ingress IP address while accessing backend pods diff --git a/tests/system-tests/rdscore/internal/rdscorecommon/hard-reboot.go b/tests/system-tests/rdscore/internal/rdscorecommon/hard-reboot.go index 45d9a15e9..0d0c01645 100644 --- a/tests/system-tests/rdscore/internal/rdscorecommon/hard-reboot.go +++ b/tests/system-tests/rdscore/internal/rdscorecommon/hard-reboot.go @@ -183,7 +183,7 @@ func WaitAllDeploymentsAreAvailable(ctx SpecContext) { // VerifySoftReboot performs graceful reboot of a cluster with cordoning and draining of individual nodes. // -//nolint:gocognit,funlen +//nolint:funlen func VerifySoftReboot(ctx SpecContext) { klog.V(rdscoreparams.RDSCoreLogLevel).Infof("\t*** Starting Soft Reboot Test Suite ***") @@ -203,6 +203,11 @@ func VerifySoftReboot(ctx SpecContext) { for _, _node := range allNodes { klog.V(rdscoreparams.RDSCoreLogLevel).Infof("Processing node %q", _node.Definition.Name) + bootIDBefore := _node.Object.Status.NodeInfo.BootID + + klog.V(rdscoreparams.RDSCoreLogLevel).Infof("Node %q boot ID before reboot: %s", + _node.Definition.Name, bootIDBefore) + klog.V(rdscoreparams.RDSCoreLogLevel).Infof("Cordoning node %q", _node.Definition.Name) err := _node.Cordon() Expect(err).ToNot(HaveOccurred(), @@ -253,30 +258,7 @@ func VerifySoftReboot(ctx SpecContext) { Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("Failed to reboot node %s", _node.Definition.Name)) - By(fmt.Sprintf("Checking node %q got into NotReady", _node.Definition.Name)) - - Eventually(func(ctx SpecContext) bool { - currentNode, err := nodes.Pull(APIClient, _node.Definition.Name) - if err != nil { - klog.V(rdscoreparams.RDSCoreLogLevel).Infof("Failed to pull node: %v", err) - - return false - } - - for _, condition := range currentNode.Object.Status.Conditions { - if condition.Type == rdscoreparams.ConditionTypeReadyString { - if condition.Status != rdscoreparams.ConstantTrueString { - klog.V(rdscoreparams.RDSCoreLogLevel).Infof("Node %q is notReady", currentNode.Definition.Name) - klog.V(rdscoreparams.RDSCoreLogLevel).Infof(" Reason: %s", condition.Reason) - - return true - } - } - } - - return false - }).WithTimeout(25*time.Minute).WithPolling(15*time.Second).WithContext(ctx).Should(BeTrue(), - "Node hasn't reached notReady state") + waitForBootIDChange(ctx, _node.Definition.Name, bootIDBefore, 25*time.Minute) By(fmt.Sprintf("Checking node %q got into Ready", _node.Definition.Name)) diff --git a/tests/system-tests/rdscore/internal/rdscorecommon/ipvlan-validation.go b/tests/system-tests/rdscore/internal/rdscorecommon/ipvlan-validation.go index f91d17741..df0ba32b4 100644 --- a/tests/system-tests/rdscore/internal/rdscorecommon/ipvlan-validation.go +++ b/tests/system-tests/rdscore/internal/rdscorecommon/ipvlan-validation.go @@ -158,33 +158,53 @@ func VerifyIPVlanOnDifferentNodes() { // VerifyIPVLANConnectivityBetweenDifferentNodes verifies connectivity between workloads, // using IPVLAN interfaces and running on different nodes. func VerifyIPVLANConnectivityBetweenDifferentNodes() { - verifySRIOVConnectivity( - RDSCoreConfig.IPVlanNSOne, - RDSCoreConfig.IPVlanNSOne, - ipvlanDeploy10Label, - ipvlanDeploy11Label, - RDSCoreConfig.IPVlanDeploy1TargetAddress) + Expect(RDSCoreConfig.IPVlanDeploy1TargetAddress != "" || + RDSCoreConfig.IPVlanDeploy1TargetAddressIPv6 != "").To(BeTrue(), + "At least one target address (IPv4 or IPv6) must be configured for IPVlan Deploy1") - verifySRIOVConnectivity( - RDSCoreConfig.IPVlanNSOne, - RDSCoreConfig.IPVlanNSOne, - ipvlanDeploy11Label, - ipvlanDeploy10Label, - RDSCoreConfig.IPVlanDeploy2TargetAddress) + addressesList := []string{RDSCoreConfig.IPVlanDeploy1TargetAddress, + RDSCoreConfig.IPVlanDeploy1TargetAddressIPv6} - verifySRIOVConnectivity( - RDSCoreConfig.IPVlanNSOne, - RDSCoreConfig.IPVlanNSOne, - ipvlanDeploy10Label, - ipvlanDeploy11Label, - RDSCoreConfig.IPVlanDeploy1TargetAddressIPv6) + for _, targetAddress := range addressesList { + if targetAddress == "" { + klog.V(rdscoreparams.RDSCoreLogLevel).Infof("Skipping empty address %q", targetAddress) - verifySRIOVConnectivity( - RDSCoreConfig.IPVlanNSOne, - RDSCoreConfig.IPVlanNSOne, - ipvlanDeploy11Label, - ipvlanDeploy10Label, - RDSCoreConfig.IPVlanDeploy2TargetAddressIPv6) + continue + } + + klog.V(rdscoreparams.RDSCoreLogLevel).Infof("Access workload via %q", targetAddress) + + verifySRIOVConnectivity( + RDSCoreConfig.IPVlanNSOne, + RDSCoreConfig.IPVlanNSOne, + ipvlanDeploy10Label, + ipvlanDeploy11Label, + targetAddress) + } + + Expect(RDSCoreConfig.IPVlanDeploy2TargetAddress != "" || + RDSCoreConfig.IPVlanDeploy2TargetAddressIPv6 != "").To(BeTrue(), + "At least one target address (IPv4 or IPv6) must be configured for IPVlan Deploy2") + + addressesList = []string{RDSCoreConfig.IPVlanDeploy2TargetAddress, + RDSCoreConfig.IPVlanDeploy2TargetAddressIPv6} + + for _, targetAddress := range addressesList { + if targetAddress == "" { + klog.V(rdscoreparams.RDSCoreLogLevel).Infof("Skipping empty address %q", targetAddress) + + continue + } + + klog.V(rdscoreparams.RDSCoreLogLevel).Infof("Access workload via %q", targetAddress) + + verifySRIOVConnectivity( + RDSCoreConfig.IPVlanNSOne, + RDSCoreConfig.IPVlanNSOne, + ipvlanDeploy11Label, + ipvlanDeploy10Label, + targetAddress) + } } // VerifyIPVlanOnSameNode verifies connectivity between freshly deployed workloads that use @@ -291,35 +311,56 @@ func VerifyIPVlanOnSameNode() { // VerifyIPVLANConnectivityOnSameNode verifies connectivity between workloads that use IPVLAN net. func VerifyIPVLANConnectivityOnSameNode() { - verifySRIOVConnectivity( - RDSCoreConfig.IPVlanNSOne, - RDSCoreConfig.IPVlanNSOne, - ipvlanDeploy20Label, - ipvlanDeploy21Label, - RDSCoreConfig.IPVlanDeploy3TargetAddress) + Expect(RDSCoreConfig.IPVlanDeploy3TargetAddress != "" || + RDSCoreConfig.IPVlanDeploy3TargetAddressIPv6 != "").To(BeTrue(), + "At least one target address (IPv4 or IPv6) must be configured for IPVlan Deploy3") - verifySRIOVConnectivity( - RDSCoreConfig.IPVlanNSOne, - RDSCoreConfig.IPVlanNSOne, - ipvlanDeploy21Label, - ipvlanDeploy20Label, - RDSCoreConfig.IPVlanDeploy4TargetAddress) + addressesList := []string{RDSCoreConfig.IPVlanDeploy3TargetAddress, + RDSCoreConfig.IPVlanDeploy3TargetAddressIPv6} - verifySRIOVConnectivity( - RDSCoreConfig.IPVlanNSOne, - RDSCoreConfig.IPVlanNSOne, - ipvlanDeploy20Label, - ipvlanDeploy21Label, - RDSCoreConfig.IPVlanDeploy3TargetAddressIPv6) + for _, targetAddress := range addressesList { + if targetAddress == "" { + klog.V(rdscoreparams.RDSCoreLogLevel).Infof("Skipping empty address %q", targetAddress) - verifySRIOVConnectivity( - RDSCoreConfig.IPVlanNSOne, - RDSCoreConfig.IPVlanNSOne, - ipvlanDeploy21Label, - ipvlanDeploy20Label, - RDSCoreConfig.IPVlanDeploy4TargetAddressIPv6) + continue + } + + klog.V(rdscoreparams.RDSCoreLogLevel).Infof("Access workload via %q", targetAddress) + + verifySRIOVConnectivity( + RDSCoreConfig.IPVlanNSOne, + RDSCoreConfig.IPVlanNSOne, + ipvlanDeploy20Label, + ipvlanDeploy21Label, + targetAddress) + } + + Expect(RDSCoreConfig.IPVlanDeploy4TargetAddress != "" || + RDSCoreConfig.IPVlanDeploy4TargetAddressIPv6 != "").To(BeTrue(), + "At least one target address (IPv4 or IPv6) must be configured for IPVlan Deploy4") + + addressesList = []string{RDSCoreConfig.IPVlanDeploy4TargetAddress, + RDSCoreConfig.IPVlanDeploy4TargetAddressIPv6} + + for _, targetAddress := range addressesList { + if targetAddress == "" { + klog.V(rdscoreparams.RDSCoreLogLevel).Infof("Skipping empty address %q", targetAddress) + + continue + } + + klog.V(rdscoreparams.RDSCoreLogLevel).Infof("Access workload via %q", targetAddress) + + verifySRIOVConnectivity( + RDSCoreConfig.IPVlanNSOne, + RDSCoreConfig.IPVlanNSOne, + ipvlanDeploy21Label, + ipvlanDeploy20Label, + targetAddress) + } } +// defineIPVlanDeployment creates an IPVLAN deployment builder with the specified configuration. func defineIPVlanDeployment(dName, nsName, dLabels, netDefName, volName string, dContainer *corev1.Container, nodeSelector map[string]string) *deployment.Builder { diff --git a/tests/system-tests/rdscore/internal/rdscorecommon/macvlan-validation.go b/tests/system-tests/rdscore/internal/rdscorecommon/macvlan-validation.go index ab3165a49..9af039178 100644 --- a/tests/system-tests/rdscore/internal/rdscorecommon/macvlan-validation.go +++ b/tests/system-tests/rdscore/internal/rdscorecommon/macvlan-validation.go @@ -203,33 +203,53 @@ func VerifyMacVlanOnDifferentNodes() { // VerifyMACVLANConnectivityBetweenDifferentNodes verifies connectivity between workloads, // using MACVLAN interfaces and running on different nodes. func VerifyMACVLANConnectivityBetweenDifferentNodes() { - verifySRIOVConnectivity( - RDSCoreConfig.MCVlanNSOne, - RDSCoreConfig.MCVlanNSOne, - macvlanDeploy10Label, - macvlanDeploy11Label, - RDSCoreConfig.MCVlanDeploy1TargetAddress) + Expect(RDSCoreConfig.MCVlanDeploy1TargetAddress != "" || + RDSCoreConfig.MCVlanDeploy1TargetAddressIPv6 != "").To(BeTrue(), + "At least one target address (IPv4 or IPv6) must be configured for MCVlan Deploy1") - verifySRIOVConnectivity( - RDSCoreConfig.MCVlanNSOne, - RDSCoreConfig.MCVlanNSOne, - macvlanDeploy11Label, - macvlanDeploy10Label, - RDSCoreConfig.MCVlanDeploy2TargetAddress) + addressesList := []string{RDSCoreConfig.MCVlanDeploy1TargetAddress, + RDSCoreConfig.MCVlanDeploy1TargetAddressIPv6} - verifySRIOVConnectivity( - RDSCoreConfig.MCVlanNSOne, - RDSCoreConfig.MCVlanNSOne, - macvlanDeploy10Label, - macvlanDeploy11Label, - RDSCoreConfig.MCVlanDeploy1TargetAddressIPv6) + for _, targetAddress := range addressesList { + if targetAddress == "" { + klog.V(rdscoreparams.RDSCoreLogLevel).Infof("Skipping empty address %q", targetAddress) - verifySRIOVConnectivity( - RDSCoreConfig.MCVlanNSOne, - RDSCoreConfig.MCVlanNSOne, - macvlanDeploy11Label, - macvlanDeploy10Label, - RDSCoreConfig.MCVlanDeploy2TargetAddressIPv6) + continue + } + + klog.V(rdscoreparams.RDSCoreLogLevel).Infof("Access workload via %q", targetAddress) + + verifySRIOVConnectivity( + RDSCoreConfig.MCVlanNSOne, + RDSCoreConfig.MCVlanNSOne, + macvlanDeploy10Label, + macvlanDeploy11Label, + targetAddress) + } + + Expect(RDSCoreConfig.MCVlanDeploy2TargetAddress != "" || + RDSCoreConfig.MCVlanDeploy2TargetAddressIPv6 != "").To(BeTrue(), + "At least one target address (IPv4 or IPv6) must be configured for MCVlan Deploy2") + + addressesList = []string{RDSCoreConfig.MCVlanDeploy2TargetAddress, + RDSCoreConfig.MCVlanDeploy2TargetAddressIPv6} + + for _, targetAddress := range addressesList { + if targetAddress == "" { + klog.V(rdscoreparams.RDSCoreLogLevel).Infof("Skipping empty address %q", targetAddress) + + continue + } + + klog.V(rdscoreparams.RDSCoreLogLevel).Infof("Access workload via %q", targetAddress) + + verifySRIOVConnectivity( + RDSCoreConfig.MCVlanNSOne, + RDSCoreConfig.MCVlanNSOne, + macvlanDeploy11Label, + macvlanDeploy10Label, + targetAddress) + } } // VerifyMacVlanOnSameNode verifies connectivity between freshly deployed workloads that use @@ -336,33 +356,53 @@ func VerifyMacVlanOnSameNode() { // VerifyMACVLANConnectivityOnSameNode verifies connectivity between workloads that use MACVLAN net. func VerifyMACVLANConnectivityOnSameNode() { - verifySRIOVConnectivity( - RDSCoreConfig.MCVlanNSOne, - RDSCoreConfig.MCVlanNSOne, - macvlanDeploy20Label, - macvlanDeploy21Label, - RDSCoreConfig.MCVlanDeploy3TargetAddress) + Expect(RDSCoreConfig.MCVlanDeploy3TargetAddress != "" || + RDSCoreConfig.MCVlanDeploy3TargetAddressIPv6 != "").To(BeTrue(), + "At least one target address (IPv4 or IPv6) must be configured for MCVlan Deploy3") - verifySRIOVConnectivity( - RDSCoreConfig.MCVlanNSOne, - RDSCoreConfig.MCVlanNSOne, - macvlanDeploy21Label, - macvlanDeploy20Label, - RDSCoreConfig.MCVlanDeploy4TargetAddress) + addressesList := []string{RDSCoreConfig.MCVlanDeploy3TargetAddress, + RDSCoreConfig.MCVlanDeploy3TargetAddressIPv6} - verifySRIOVConnectivity( - RDSCoreConfig.MCVlanNSOne, - RDSCoreConfig.MCVlanNSOne, - macvlanDeploy20Label, - macvlanDeploy21Label, - RDSCoreConfig.MCVlanDeploy3TargetAddressIPv6) + for _, targetAddress := range addressesList { + if targetAddress == "" { + klog.V(rdscoreparams.RDSCoreLogLevel).Infof("Skipping empty address %q", targetAddress) - verifySRIOVConnectivity( - RDSCoreConfig.MCVlanNSOne, - RDSCoreConfig.MCVlanNSOne, - macvlanDeploy21Label, - macvlanDeploy20Label, - RDSCoreConfig.MCVlanDeploy4TargetAddressIPv6) + continue + } + + klog.V(rdscoreparams.RDSCoreLogLevel).Infof("Access workload via %q", targetAddress) + + verifySRIOVConnectivity( + RDSCoreConfig.MCVlanNSOne, + RDSCoreConfig.MCVlanNSOne, + macvlanDeploy20Label, + macvlanDeploy21Label, + targetAddress) + } + + Expect(RDSCoreConfig.MCVlanDeploy4TargetAddress != "" || + RDSCoreConfig.MCVlanDeploy4TargetAddressIPv6 != "").To(BeTrue(), + "At least one target address (IPv4 or IPv6) must be configured for MCVlan Deploy4") + + addressesList = []string{RDSCoreConfig.MCVlanDeploy4TargetAddress, + RDSCoreConfig.MCVlanDeploy4TargetAddressIPv6} + + for _, targetAddress := range addressesList { + if targetAddress == "" { + klog.V(rdscoreparams.RDSCoreLogLevel).Infof("Skipping empty address %q", targetAddress) + + continue + } + + klog.V(rdscoreparams.RDSCoreLogLevel).Infof("Access workload via %q", targetAddress) + + verifySRIOVConnectivity( + RDSCoreConfig.MCVlanNSOne, + RDSCoreConfig.MCVlanNSOne, + macvlanDeploy21Label, + macvlanDeploy20Label, + targetAddress) + } } func defineMacVlanDeployment(dName, nsName, dLabels, netDefName, volName string, diff --git a/tests/system-tests/rdscore/internal/rdscorecommon/monitoring-config-validation.go b/tests/system-tests/rdscore/internal/rdscorecommon/monitoring-config-validation.go index 935a754b1..275e8a459 100644 --- a/tests/system-tests/rdscore/internal/rdscorecommon/monitoring-config-validation.go +++ b/tests/system-tests/rdscore/internal/rdscorecommon/monitoring-config-validation.go @@ -490,8 +490,17 @@ class RemoteWriteHandler(http.server.BaseHTTPRequestHandler): def log_message(self, format, *args): pass # Suppress default logging +import socket + +class DualStackTCPServer(socketserver.TCPServer): + address_family = socket.AF_INET6 + + def server_bind(self): + self.socket.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 0) + super().server_bind() + PORT = %d -with socketserver.TCPServer(("", PORT), RemoteWriteHandler) as httpd: +with DualStackTCPServer(("::", PORT), RemoteWriteHandler) as httpd: httpd.serve_forever() `, remoteWriteTestContainerPort) } diff --git a/tests/system-tests/rdscore/internal/rdscorecommon/nmi-redfish.go b/tests/system-tests/rdscore/internal/rdscorecommon/nmi-redfish.go index 2bdb586e7..1706ce93a 100644 --- a/tests/system-tests/rdscore/internal/rdscorecommon/nmi-redfish.go +++ b/tests/system-tests/rdscore/internal/rdscorecommon/nmi-redfish.go @@ -61,6 +61,13 @@ func triggerNMIRedfish(ctx SpecContext, nodeLabel string) { By(fmt.Sprintf("Cleaning up /var/crash directory on node %q", node.Definition.Name)) cleanupVarCrashDirectory(ctx, node.Definition.Name) + By(fmt.Sprintf("Recording boot ID for node %q before NMI", node.Definition.Name)) + + bootIDBefore := node.Object.Status.NodeInfo.BootID + + klog.V(rdscoreparams.RDSCoreLogLevel).Infof("Node %q boot ID before NMI: %s", + node.Definition.Name, bootIDBefore) + By(fmt.Sprintf("Trigger NMI via RedFish on node %q", node.Definition.Name)) klog.V(rdscoreparams.RDSCoreLogLevel).Infof("Triggering NMI via RedFish on %q", node.Definition.Name) @@ -103,7 +110,7 @@ func triggerNMIRedfish(ctx SpecContext, nodeLabel string) { Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("Failed to trigger NMI on node %s", node.Definition.Name)) - waitForNodeToBeNotReady(ctx, node.Definition.Name, 15*time.Second, 25*time.Minute) + waitForBootIDChange(ctx, node.Definition.Name, bootIDBefore, 25*time.Minute) By(fmt.Sprintf("Waiting for node %q to return to Ready state", node.Definition.Name)) diff --git a/tests/system-tests/rdscore/internal/rdscorecommon/sriov-pod-level-bond.go b/tests/system-tests/rdscore/internal/rdscorecommon/sriov-pod-level-bond.go index 48c20c38a..a8f13c5f4 100644 --- a/tests/system-tests/rdscore/internal/rdscorecommon/sriov-pod-level-bond.go +++ b/tests/system-tests/rdscore/internal/rdscorecommon/sriov-pod-level-bond.go @@ -238,6 +238,7 @@ func definePodLevelBondDeploymentContainer() *pod.ContainerBuilder { return deploymentContainer } +//nolint:funlen func definePodLevelBondTestPodDeployment( apiClient *clients.Settings, containerConfig *corev1.Container, @@ -255,32 +256,66 @@ func definePodLevelBondTestPodDeployment( deployLabels map[string]string) (*deployment.Builder, error) { klog.V(100).Infof("Defining deployment %q in %q ns", deploymentName, nsName) - if bondInfIPv4 == "" { - klog.V(100).Infof("Bond interface IPv4 address is missing") + // Validate that at least one IP family is configured + if bondInfIPv4 == "" && bondInfIPv6 == "" { + klog.V(100).Infof("Both IPv4 and IPv6 bond interface addresses are missing") - return nil, fmt.Errorf("bond interface IPv4 address is missing") + return nil, fmt.Errorf("at least one bond interface IP address (IPv4 or IPv6) must be configured") } - if bondInfIPv6 == "" { - klog.V(100).Infof("Bond interface IPv6 address is missing") + // Validate IPv4 configuration: IP and subnet mask must both be present or both be absent + if bondInfIPv4 != "" && bondInfSubMaskIPv4 == "" { + klog.V(100).Infof("Bond interface IPv4 address subnet mask is missing") - return nil, fmt.Errorf("bond interface IPv6 address is missing") + return nil, fmt.Errorf("bond interface IPv4 address subnet mask is required when IPv4 address is provided") } - if bondInfSubMaskIPv4 == "" { - klog.V(100).Infof("Bond interface IPv4 address subnet mask is missing") + if bondInfIPv4 == "" && bondInfSubMaskIPv4 != "" { + klog.V(100).Infof("Bond interface IPv4 subnet mask is set without an IPv4 address") - return nil, fmt.Errorf("bond interface IPv4 address subnet mask is missing") + return nil, fmt.Errorf("bond interface IPv4 subnet mask %q is set but IPv4 address is empty", bondInfSubMaskIPv4) } - if bondInfSubMaskIPv6 == "" { + // Validate IPv6 configuration: IP and subnet mask must both be present or both be absent + if bondInfIPv6 != "" && bondInfSubMaskIPv6 == "" { klog.V(100).Infof("Bond interface IPv6 address subnet mask is missing") - return nil, fmt.Errorf("bond interface IPv6 address subnet mask is missing") + return nil, fmt.Errorf("bond interface IPv6 address subnet mask is required when IPv6 address is provided") + } + + if bondInfIPv6 == "" && bondInfSubMaskIPv6 != "" { + klog.V(100).Infof("Bond interface IPv6 subnet mask is set without an IPv6 address") + + return nil, fmt.Errorf("bond interface IPv6 subnet mask %q is set but IPv6 address is empty", bondInfSubMaskIPv6) + } + + // Log configuration status + if bondInfIPv4 == "" { + klog.V(100).Infof("IPv4 bond interface address not configured - pure IPv6 mode") + } else { + klog.V(100).Infof("IPv4 bond interface address configured: %s/%s", bondInfIPv4, bondInfSubMaskIPv4) + } + + if bondInfIPv6 == "" { + klog.V(100).Infof("IPv6 bond interface address not configured - pure IPv4 mode") + } else { + klog.V(100).Infof("IPv6 bond interface address configured: %s/%s", bondInfIPv6, bondInfSubMaskIPv6) } nodeSelector := map[string]string{"kubernetes.io/hostname": scheduleOnHost} + // Build IP requests only for configured addresses + var ipRequests []string + if bondInfIPv4 != "" && bondInfSubMaskIPv4 != "" { + ipRequests = append(ipRequests, fmt.Sprintf("%s/%s", bondInfIPv4, bondInfSubMaskIPv4)) + klog.V(100).Infof("Added IPv4 request: %s/%s", bondInfIPv4, bondInfSubMaskIPv4) + } + + if bondInfIPv6 != "" && bondInfSubMaskIPv6 != "" { + ipRequests = append(ipRequests, fmt.Sprintf("%s/%s", bondInfIPv6, bondInfSubMaskIPv6)) + klog.V(100).Infof("Added IPv6 request: %s/%s", bondInfIPv6, bondInfSubMaskIPv6) + } + netAnnotations := []*types.NetworkSelectionElement{ { Name: sriovNet1Name, @@ -291,10 +326,9 @@ func definePodLevelBondTestPodDeployment( Namespace: nsName, }, { - Name: bondNetName, - Namespace: nsName, - IPRequest: []string{fmt.Sprintf("%s/%s", bondInfIPv4, bondInfSubMaskIPv4), - fmt.Sprintf("%s/%s", bondInfIPv6, bondInfSubMaskIPv6)}, + Name: bondNetName, + Namespace: nsName, + IPRequest: ipRequests, MacRequest: bondInfMacAddr, }, } @@ -1336,21 +1370,37 @@ func VerifyPodLevelBondWorkloadsAfterVFFailOver() { fmt.Sprintf("Failed to retrieve bond active interface for the pod deployment %s in namespace %s: %v", serverPodObj.Definition.Name, serverPodObj.Definition.Namespace, err)) + // Determine target IP based on configuration priority: IPv4 first, then IPv6 + var targetIP string + + switch { + case RDSCoreConfig.PodLevelBondDeploymentTwoIPv4 != "": + targetIP = RDSCoreConfig.PodLevelBondDeploymentTwoIPv4 + case RDSCoreConfig.PodLevelBondDeploymentTwoIPv6 != "": + targetIP = RDSCoreConfig.PodLevelBondDeploymentTwoIPv6 + default: + Fail("Neither IPv4 nor IPv6 server address is configured for the server deployment") + + return + } + go func() { defer GinkgoRecover() - By("Send data from the client container to the IPv4 address used by the server container") + By(fmt.Sprintf("Send data from the client container to the server address %s", targetIP)) + + klog.V(100).Infof("Using server address %s for TCP traffic generation", targetIP) output, err := generateTCPTraffic( clientPodObj, - RDSCoreConfig.PodLevelBondDeploymentTwoIPv4, + targetIP, RDSCoreConfig.PodLevelBondPort, "10", "5") Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("Failed to generate TCP traffic from the pod %s in namespace %s to the server %s: %v", clientPodObj.Definition.Name, clientPodObj.Definition.Namespace, - RDSCoreConfig.PodLevelBondDeploymentTwoIPv4, err)) + targetIP, err)) testPassed, err := scanClientPodTrafficOutput(output) Expect(err).ToNot(HaveOccurred(), diff --git a/tests/system-tests/rdscore/internal/rdscorecommon/whereabouts-statefulset.go b/tests/system-tests/rdscore/internal/rdscorecommon/whereabouts-statefulset.go index adb688564..9169e6b9d 100644 --- a/tests/system-tests/rdscore/internal/rdscorecommon/whereabouts-statefulset.go +++ b/tests/system-tests/rdscore/internal/rdscorecommon/whereabouts-statefulset.go @@ -6,6 +6,7 @@ import ( "context" "fmt" "math/rand" + "regexp" "strconv" "strings" "time" @@ -26,6 +27,8 @@ import ( "github.com/rh-ecosystem-edge/eco-goinfra/pkg/statefulset" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/klog/v2" ) @@ -141,6 +144,7 @@ var ( } ) +// cleanupStatefulset removes a statefulset and waits for its pods to be deleted. func cleanupStatefulset(stName, namespace, stLabel string) { By(fmt.Sprintf("Checking that statefulset %q doesn't exist in %q namespace", stName, namespace)) @@ -181,6 +185,7 @@ func cleanupStatefulset(stName, namespace, stLabel string) { } } +// createStatefulsetAndWaitReplicasReady creates a statefulset and waits for all replicas to become ready. func createStatefulsetAndWaitReplicasReady(stName, namespace string, stBuilder *statefulset.Builder) { By(fmt.Sprintf("Creating statefulset %q in %q namespace", stName, namespace)) @@ -218,7 +223,121 @@ func createStatefulsetAndWaitReplicasReady(stName, namespace string, stBuilder * "Statefulset %q in %q namespace is not ready", stName, namespace) } -func setupHeadlessService(svcName, namespace, svcLabel, svcPort string) { +// determineIPFamilyPolicy fetches the NAD and inspects the IPAM range fields to determine +// whether to use RequireDualStack, or SingleStack with IPv4 or IPv6. +func determineIPFamilyPolicy(nadName, namespace string) ([]corev1.IPFamily, corev1.IPFamilyPolicy) { + nadObj, err := APIClient.Resource( + schema.GroupVersionResource{ + Group: "k8s.cni.cncf.io", + Version: "v1", + Resource: "network-attachment-definitions", + }).Namespace(namespace).Get(context.TODO(), nadName, metav1.GetOptions{}) + + Expect(err).ToNot(HaveOccurred(), + fmt.Sprintf("Failed to get NAD %q in %q namespace", nadName, namespace)) + + config, found, err := unstructured.NestedString(nadObj.Object, "spec", "config") + Expect(err).ToNot(HaveOccurred(), + fmt.Sprintf("Failed to read config from NAD %q", nadName)) + Expect(found).To(BeTrue(), + fmt.Sprintf("NAD %q has no spec.config field", nadName)) + + ranges := extractIPAMRanges(config) + Expect(len(ranges)).ToNot(Equal(0), + fmt.Sprintf("NAD %q has no IPAM range entries in spec.config", nadName)) + + hasIPv4, hasIPv6 := detectIPFamiliesFromRanges(ranges) + + klog.V(rdscoreparams.RDSCoreLogLevel).Infof("NAD %q IP family detection: hasIPv4=%v, hasIPv6=%v (ranges: %v)", + nadName, hasIPv4, hasIPv6, ranges) + + switch { + case hasIPv4 && hasIPv6: + return []corev1.IPFamily{corev1.IPv4Protocol, corev1.IPv6Protocol}, corev1.IPFamilyPolicyRequireDualStack + case hasIPv6: + return []corev1.IPFamily{corev1.IPv6Protocol}, corev1.IPFamilyPolicySingleStack + case hasIPv4: + return []corev1.IPFamily{corev1.IPv4Protocol}, corev1.IPFamilyPolicySingleStack + default: + Fail(fmt.Sprintf("NAD %q IPAM ranges contain no detectable IPv4 or IPv6 CIDR: %v", nadName, ranges)) + + return nil, "" + } +} + +// extractIPAMRanges parses the NAD spec.config JSON and returns all IPAM range strings. +// It supports both top-level ipam config and plugins[*].ipam config layouts, +// with both ipam.range (single) and ipam.ipRanges[*].range (multiple) forms. +func extractIPAMRanges(config string) []string { + var parsed map[string]interface{} + if err := json.Unmarshal([]byte(config), &parsed); err != nil { + klog.V(rdscoreparams.RDSCoreLogLevel).Infof("Failed to parse NAD config JSON: %v", err) + + return nil + } + + var ranges []string + + ranges = append(ranges, extractRangesFromIPAM(parsed)...) + + if plugins, ok := parsed["plugins"].([]interface{}); ok { + for _, p := range plugins { + if plugin, ok := p.(map[string]interface{}); ok { + ranges = append(ranges, extractRangesFromIPAM(plugin)...) + } + } + } + + return ranges +} + +// extractRangesFromIPAM extracts range strings from an object's ipam field. +func extractRangesFromIPAM(obj map[string]interface{}) []string { + ipam, ok := obj["ipam"].(map[string]interface{}) + if !ok { + return nil + } + + var ranges []string + + if r, ok := ipam["range"].(string); ok { + ranges = append(ranges, r) + } + + if ipRanges, ok := ipam["ipRanges"].([]interface{}); ok { + for _, entry := range ipRanges { + if rangeMap, ok := entry.(map[string]interface{}); ok { + if r, ok := rangeMap["range"].(string); ok { + ranges = append(ranges, r) + } + } + } + } + + return ranges +} + +// detectIPFamiliesFromRanges inspects a list of CIDR range strings and returns +// whether IPv4 and/or IPv6 ranges are present. +func detectIPFamiliesFromRanges(ranges []string) (hasIPv4, hasIPv6 bool) { + ipv4Re := regexp.MustCompile(`\d+\.\d+\.\d+\.\d+/\d+`) + ipv6Re := regexp.MustCompile(`([0-9a-fA-F]{0,4}:){2,}[0-9a-fA-F]{0,4}/\d+`) + + for _, r := range ranges { + if ipv4Re.MatchString(r) { + hasIPv4 = true + } + + if ipv6Re.MatchString(r) { + hasIPv6 = true + } + } + + return hasIPv4, hasIPv6 +} + +// setupHeadlessService creates a headless service with ipFamilyPolicy determined from the NAD configuration. +func setupHeadlessService(svcName, namespace, svcLabel, svcPort, nadName string) { By(fmt.Sprintf("Checking that service %q doesn't exist in %q namespace", svcName, namespace)) @@ -260,12 +379,14 @@ func setupHeadlessService(svcName, namespace, svcLabel, svcPort string) { svcOne = defineHeadlessService(svcName, namespace, svcLabelsMap, svcPortCr) - By("Setting ipFamilyPolicy to 'RequireDualStack'") + ipFamilies, ipFamilyPolicy := determineIPFamilyPolicy(nadName, namespace) + + By(fmt.Sprintf("Setting ipFamilyPolicy to %q for NAD %q", ipFamilyPolicy, nadName)) - klog.V(rdscoreparams.RDSCoreLogLevel).Infof("Setting ipFamilyPolicy to 'RequireDualStack'") + klog.V(rdscoreparams.RDSCoreLogLevel).Infof("NAD %q: ipFamilies=%v, ipFamilyPolicy=%s", + nadName, ipFamilies, ipFamilyPolicy) - svcOne = svcOne.WithIPFamily([]corev1.IPFamily{"IPv4", "IPv6"}, - corev1.IPFamilyPolicyRequireDualStack) + svcOne = svcOne.WithIPFamily(ipFamilies, ipFamilyPolicy) By(fmt.Sprintf("Creating headless service %q in %q namespace", svcName, namespace)) @@ -285,6 +406,7 @@ func setupHeadlessService(svcName, namespace, svcLabel, svcPort string) { "Failed to create headless service %q in %q namespace", svcName, namespace) } +// verifyInterPodCommunication validates network connectivity between all active pods via their whereabouts IPs. func verifyInterPodCommunication( activePods []*pod.Builder, podWhereaboutsIPs map[string][]NetworkInterface, @@ -597,6 +719,7 @@ func getActivePods(podLabel, namespace string) []*pod.Builder { return activePods } +// ensurePodConnectivityAfterPodTermination verifies inter-pod connectivity is restored after terminating a pod. func ensurePodConnectivityAfterPodTermination(stLabel, namespace, targetPort string, stReplicas int) { By("Getting list of active pods") @@ -656,6 +779,8 @@ func ensurePodConnectivityAfterPodTermination(stLabel, namespace, targetPort str VerifyPodConnectivity(stLabel, namespace, interfaceName, parsedPort) } +// ensurePodConnectivityAfterNodeDrain verifies inter-pod connectivity is restored after draining a node. +// //nolint:funlen func ensurePodConnectivityAfterNodeDrain(stLabel, namespace, targetPort string, stReplicas int, sameNode bool) { By("Getting list of active pods") @@ -755,6 +880,7 @@ func ensurePodConnectivityAfterNodeDrain(stLabel, namespace, targetPort string, VerifyPodConnectivity(stLabel, namespace, interfaceName, parsedPort) } +// powerOnNodeWaitReady powers on a node via BMC and waits for it to reach Ready state. func powerOnNodeWaitReady(bmcClient *bmc.BMC, nodeToPowerOff string, stopCh chan bool) { By("Stopping keepNodePoweredOff goroutine") @@ -823,6 +949,7 @@ func powerOnNodeWaitReady(bmcClient *bmc.BMC, nodeToPowerOff string, stopCh chan klog.V(rdscoreparams.RDSCoreLogLevel).Infof("Successfully powered on %q", nodeToPowerOff) } +// keepNodePoweredOff continuously monitors and powers off a node via BMC until signaled to stop. func keepNodePoweredOff(bmcClient *bmc.BMC, nodeToPowerOff string, timeout time.Duration, stopCh chan bool) { By(fmt.Sprintf("Keeping node %q powered off", nodeToPowerOff)) @@ -876,6 +1003,8 @@ func keepNodePoweredOff(bmcClient *bmc.BMC, nodeToPowerOff string, timeout time. klog.V(rdscoreparams.RDSCoreLogLevel).Infof("keepNodePoweredOff finished") } +// ensurePodConnectivityAfterNodePowerOff verifies inter-pod connectivity is restored after powering off a node. +// //nolint:gocognit,funlen func ensurePodConnectivityAfterNodePowerOff(stLabel, namespace, targetPort string, stReplicas int, sameNode bool) { By("Getting list of active pods") @@ -1128,7 +1257,7 @@ func CreateWhereaboutsStatefulset(ctx SpecContext, config StatefulsetConfig) { configureWhereaboutsIPReconciler() // Setup headless service - setupHeadlessService(config.ServiceName, RDSCoreConfig.WhereaboutNS, config.Label, config.Port) + setupHeadlessService(config.ServiceName, RDSCoreConfig.WhereaboutNS, config.Label, config.Port, config.NAD) // Cleanup existing statefulset cleanupStatefulset(config.Name, RDSCoreConfig.WhereaboutNS, config.Label)