Skip to content

Commit 7c5836a

Browse files
zhaocongqizhaocongqi
authored
Release 1.12.12 mc (#6112)
* fix(vm/sts): handle multiple network interfaces when cleaning resources on VM Pod deletion (#6089) Signed-off-by: zhaocongqi <1229896069@qq.com> * update mc version Signed-off-by: zhaocongqi <1229896069@qq.com> --------- Signed-off-by: zhaocongqi <1229896069@qq.com>
1 parent 6731fff commit 7c5836a

File tree

2 files changed

+94
-47
lines changed

2 files changed

+94
-47
lines changed

‎VERSION‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
v1.12.11-mc
1+
v1.12.12-mc

‎pkg/controller/pod.go‎

Lines changed: 93 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -1043,23 +1043,25 @@ func (c *Controller) handleDeletePod(key string) error {
10431043

10441044
podKey := fmt.Sprintf("%s/%s", pod.Namespace, podName)
10451045

1046-
var keepIPCR bool
1047-
if ok, stsName, stsUID := isStatefulSetPod(pod); ok {
1048-
if pod.DeletionTimestamp != nil {
1049-
klog.Infof("handle deletion of sts pod %s", podName)
1050-
toDel := isStatefulSetPodToDel(c.config.KubeClient, pod, stsName, stsUID)
1051-
if !toDel {
1046+
var keepIPCR, isOwnerRefToDel, isOwnerRefDeleted bool
1047+
var ipcrToDelete []string
1048+
isStsPod, stsName, stsUID := isStatefulSetPod(pod)
1049+
if isStsPod {
1050+
if !pod.DeletionTimestamp.IsZero() {
1051+
klog.Infof("handle deletion of sts pod %s", podKey)
1052+
isOwnerRefToDel = isStatefulSetPodToDel(c.config.KubeClient, pod, stsName, stsUID)
1053+
if !isOwnerRefToDel {
10521054
klog.Infof("try keep ip for sts pod %s", podKey)
10531055
keepIPCR = true
10541056
}
10551057
}
10561058
if keepIPCR {
1057-
isDelete, err := appendCheckPodToDel(c, pod, stsName, util.StatefulSet)
1059+
isOwnerRefDeleted, ipcrToDelete, err := appendCheckPodNetToDel(c, pod, stsName, util.StatefulSet)
10581060
if err != nil {
10591061
klog.Error(err)
10601062
return err
10611063
}
1062-
if isDelete {
1064+
if isOwnerRefDeleted || len(ipcrToDelete) != 0 {
10631065
klog.Infof("not keep ip for sts pod %s", podKey)
10641066
keepIPCR = false
10651067
}
@@ -1080,20 +1082,20 @@ func (c *Controller) handleDeletePod(key string) error {
10801082
}
10811083
}
10821084
if pod.DeletionTimestamp != nil {
1083-
klog.Infof("handle deletion of vm pod %s", podName)
1084-
vmToBeDel := c.isVMToDel(pod, vmName)
1085-
if !vmToBeDel {
1085+
klog.Infof("handle deletion of vm pod %s", podKey)
1086+
isOwnerRefToDel = c.isVMToDel(pod, vmName)
1087+
if !isOwnerRefToDel {
10861088
klog.Infof("try keep ip for vm pod %s", podKey)
10871089
keepIPCR = true
10881090
}
10891091
}
10901092
if keepIPCR {
1091-
isDelete, err := appendCheckPodToDel(c, pod, vmName, util.VMInstance)
1093+
isOwnerRefDeleted, ipcrToDelete, err = appendCheckPodNetToDel(c, pod, vmName, util.VMInstance)
10921094
if err != nil {
10931095
klog.Error(err)
10941096
return err
10951097
}
1096-
if isDelete {
1098+
if isOwnerRefDeleted || len(ipcrToDelete) != 0 {
10971099
klog.Infof("not keep ip for vm pod %s", podKey)
10981100
keepIPCR = false
10991101
}
@@ -1170,6 +1172,12 @@ func (c *Controller) handleDeletePod(key string) error {
11701172
klog.Infof("try release all ip address for deleting pod %s", podKey)
11711173
for _, podNet := range podNets {
11721174
portName := ovs.PodNameToPortName(podName, pod.Namespace, podNet.ProviderName)
1175+
// if the OwnerRef has been deleted or is in the process of being deleted, all associated IPCRs must be cleaned up
1176+
if (isStsPod || isVMPod) && !isOwnerRefToDel && !isOwnerRefDeleted &&
1177+
!slices.Contains(ipcrToDelete, portName) {
1178+
klog.Infof("skip clean ip CR %s", portName)
1179+
continue
1180+
}
11731181
ipCR, err := c.ipsLister.Get(portName)
11741182
if err != nil {
11751183
if k8serrors.IsNotFound(err) {
@@ -1967,93 +1975,132 @@ func (c *Controller) acquireStaticAddress(key, nicName, ip string, mac *string,
19671975
return v4IP, v6IP, macStr, nil
19681976
}
19691977

1970-
func appendCheckPodToDel(c *Controller, pod *v1.Pod, ownerRefName, ownerRefKind string) (bool, error) {
1971-
// subnet for ns has been changed, and statefulset pod's ip is not in the range of subnet's cidr anymore
1978+
func appendCheckPodNetToDel(c *Controller, pod *v1.Pod, ownerRefName, ownerRefKind string) (bool, []string, error) {
1979+
// subnet for ns has been changed, and statefulset/vm pod's ip is not in the range of subnet's cidr anymore
19721980
podNs, err := c.namespacesLister.Get(pod.Namespace)
19731981
if err != nil {
19741982
klog.Errorf("failed to get namespace %s, %v", pod.Namespace, err)
1975-
return false, err
1983+
return false, nil, err
19761984
}
19771985

1978-
// check if subnet exist in OwnerReference
1979-
var ownerRefSubnetExist bool
1980-
var ownerRefSubnet string
1986+
var ownerRefAnnotations map[string]string
19811987
switch ownerRefKind {
19821988
case util.StatefulSet:
19831989
ss, err := c.config.KubeClient.AppsV1().StatefulSets(pod.Namespace).Get(context.Background(), ownerRefName, metav1.GetOptions{})
19841990
if err != nil {
19851991
if k8serrors.IsNotFound(err) {
19861992
klog.Infof("Statefulset %s is not found", ownerRefName)
1987-
return true, nil
1993+
return true, nil, nil
19881994
}
19891995
klog.Errorf("failed to get StatefulSet %s, %v", ownerRefName, err)
19901996
}
1991-
if ss.Spec.Template.ObjectMeta.Annotations[util.LogicalSwitchAnnotation] != "" {
1992-
ownerRefSubnetExist = true
1993-
ownerRefSubnet = ss.Spec.Template.ObjectMeta.Annotations[util.LogicalSwitchAnnotation]
1997+
if ss.Spec.Template.Annotations != nil {
1998+
ownerRefAnnotations = ss.Spec.Template.Annotations
19941999
}
19952000

19962001
case util.VMInstance:
19972002
vm, err := c.config.KubevirtClient.VirtualMachine(pod.Namespace).Get(context.Background(), ownerRefName, metav1.GetOptions{})
19982003
if err != nil {
19992004
if k8serrors.IsNotFound(err) {
20002005
klog.Infof("VirtualMachine %s is not found", ownerRefName)
2001-
return true, nil
2006+
return true, nil, nil
20022007
}
20032008
klog.Errorf("failed to get VirtualMachine %s, %v", ownerRefName, err)
20042009
}
20052010
if vm != nil &&
20062011
vm.Spec.Template != nil &&
2007-
vm.Spec.Template.ObjectMeta.Annotations != nil &&
2008-
vm.Spec.Template.ObjectMeta.Annotations[util.LogicalSwitchAnnotation] != "" {
2009-
ownerRefSubnetExist = true
2010-
ownerRefSubnet = vm.Spec.Template.ObjectMeta.Annotations[util.LogicalSwitchAnnotation]
2011-
}
2012-
}
2013-
podSwitch := strings.TrimSpace(pod.Annotations[util.LogicalSwitchAnnotation])
2014-
if !ownerRefSubnetExist {
2015-
nsSubnetNames := podNs.Annotations[util.LogicalSwitchAnnotation]
2016-
// check if pod use the subnet of its ns
2017-
if nsSubnetNames != "" && podSwitch != "" && !slices.Contains(strings.Split(nsSubnetNames, ","), podSwitch) {
2018-
klog.Infof("ns %s annotation subnet is %s, which is inconstant with subnet for pod %s, delete pod", pod.Namespace, nsSubnetNames, pod.Name)
2019-
return true, nil
2012+
vm.Spec.Template.ObjectMeta.Annotations != nil {
2013+
ownerRefAnnotations = vm.Spec.Template.ObjectMeta.Annotations
20202014
}
20212015
}
20222016

2017+
var ipcrToDelete []string
2018+
if defaultIPCRName := appendCheckPodNonMultusNetToDel(c, pod, ownerRefName, ownerRefAnnotations, podNs); defaultIPCRName != "" {
2019+
ipcrToDelete = append(ipcrToDelete, defaultIPCRName)
2020+
}
2021+
2022+
if multusIPCRNames := appendCheckPodMultusNetToDel(c, pod, ownerRefName, ownerRefAnnotations); len(multusIPCRNames) != 0 {
2023+
ipcrToDelete = append(ipcrToDelete, multusIPCRNames...)
2024+
}
2025+
2026+
return false, ipcrToDelete, nil
2027+
}
2028+
2029+
func appendCheckPodNonMultusNetToDel(c *Controller, pod *v1.Pod, ownerRefName string, ownerRefAnnotations map[string]string, podNs *v1.Namespace) string {
2030+
podDefaultSwitch := strings.TrimSpace(pod.Annotations[util.LogicalSwitchAnnotation])
2031+
if podDefaultSwitch != "" {
2032+
ownerRefSubnet := ownerRefAnnotations[util.LogicalSwitchAnnotation]
2033+
defaultIPCRName := ovs.PodNameToPortName(ownerRefName, pod.Namespace, util.OvnProvider)
2034+
if ownerRefSubnet == "" {
2035+
nsSubnetNames := podNs.Annotations[util.LogicalSwitchAnnotation]
2036+
// check if pod use the subnet of its ns
2037+
if nsSubnetNames != "" && !slices.Contains(strings.Split(nsSubnetNames, ","), podDefaultSwitch) {
2038+
klog.Infof("ns %s annotation subnet is %s, which is inconstant with subnet for pod %s, delete pod", pod.Namespace, nsSubnetNames, pod.Name)
2039+
return defaultIPCRName
2040+
}
2041+
} else {
2042+
podIP := pod.Annotations[util.IPAddressAnnotation]
2043+
if shouldCleanPodNet(c, pod, ownerRefName, ownerRefSubnet, podDefaultSwitch, podIP) {
2044+
return defaultIPCRName
2045+
}
2046+
}
2047+
}
2048+
return ""
2049+
}
2050+
2051+
func appendCheckPodMultusNetToDel(c *Controller, pod *v1.Pod, ownerRefName string, ownerRefAnnotations map[string]string) []string {
2052+
var multusIPCRNames []string
2053+
attachmentNets, _ := c.getPodAttachmentNet(pod)
2054+
for _, attachmentNet := range attachmentNets {
2055+
ipCRName := ovs.PodNameToPortName(ownerRefName, pod.Namespace, attachmentNet.ProviderName)
2056+
podSwitch := strings.TrimSpace(pod.Annotations[fmt.Sprintf(util.LogicalSwitchAnnotationTemplate, attachmentNet.ProviderName)])
2057+
ownerRefSubnet := ownerRefAnnotations[fmt.Sprintf(util.LogicalSwitchAnnotationTemplate, attachmentNet.ProviderName)]
2058+
podIP := pod.Annotations[fmt.Sprintf(util.IPAddressAnnotationTemplate, attachmentNet.ProviderName)]
2059+
if shouldCleanPodNet(c, pod, ownerRefName, ownerRefSubnet, podSwitch, podIP) {
2060+
multusIPCRNames = append(multusIPCRNames, ipCRName)
2061+
}
2062+
}
2063+
return multusIPCRNames
2064+
}
2065+
2066+
func shouldCleanPodNet(c *Controller, pod *v1.Pod, ownerRefName, ownerRefSubnet, podSwitch, podIP string) bool {
20232067
// subnet cidr has been changed, and statefulset pod's ip is not in the range of subnet's cidr anymore
20242068
podSubnet, err := c.subnetsLister.Get(podSwitch)
20252069
if err != nil {
2070+
if k8serrors.IsNotFound(err) {
2071+
klog.Infof("subnet %s not found for pod %s/%s, not auto clean ip", podSwitch, pod.Namespace, pod.Name)
2072+
return false
2073+
}
20262074
klog.Errorf("failed to get subnet %s, %v, not auto clean ip", podSwitch, err)
2027-
return false, err
2075+
return false
20282076
}
20292077
if podSubnet == nil {
20302078
// TODO: remove: CRD get interface will retrun a nil subnet ?
20312079
klog.Errorf("pod %s/%s subnet %s is nil, not auto clean ip", pod.Namespace, pod.Name, podSwitch)
2032-
return false, nil
2080+
return false
20332081
}
2034-
podIP := pod.Annotations[util.IPAddressAnnotation]
20352082
if podIP == "" {
20362083
// delete pod just after it created < 1ms
20372084
klog.Infof("pod %s/%s annotaions has no ip address, not auto clean ip", pod.Namespace, pod.Name)
2038-
return false, nil
2085+
return false
20392086
}
20402087
podSubnetCidr := podSubnet.Spec.CIDRBlock
20412088
if podSubnetCidr == "" {
20422089
// subnet spec cidr changed by user
20432090
klog.Errorf("invalid pod subnet %s empty cidr %s, not auto clean ip", podSwitch, podSubnetCidr)
2044-
return false, nil
2091+
return false
20452092
}
20462093
if !util.CIDRContainIP(podSubnetCidr, podIP) {
2047-
klog.Infof("pod's ip %s is not in the range of subnet %s, delete pod", pod.Annotations[util.IPAddressAnnotation], podSubnet.Name)
2048-
return true, nil
2094+
klog.Infof("pod's ip %s is not in the range of subnet %s, delete pod", podIP, podSubnet.Name)
2095+
return true
20492096
}
20502097
// subnet of ownerReference(sts/vm) has been changed, it needs to handle delete pod and create port on the new logical switch
20512098
if ownerRefSubnet != "" && podSubnet.Name != ownerRefSubnet {
20522099
klog.Infof("Subnet of owner %s has been changed from %s to %s, delete pod %s/%s", ownerRefName, podSubnet.Name, ownerRefSubnet, pod.Namespace, pod.Name)
2053-
return true, nil
2100+
return true
20542101
}
20552102

2056-
return false, nil
2103+
return false
20572104
}
20582105

20592106
func isVMPod(pod *v1.Pod) (bool, string) {

0 commit comments

Comments
 (0)