Skip to content

Commit 564a29e

Browse files
authored
controller: fix ip allocation from IPPools (#5729)
Signed-off-by: zhangzujian <zhangzujian.7@gmail.com>
1 parent 6f15735 commit 564a29e

File tree

6 files changed

+96
-17
lines changed

6 files changed

+96
-17
lines changed

.github/workflows/build-x86-image.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1048,7 +1048,7 @@ jobs:
10481048
- build-kube-ovn
10491049
- build-e2e-binaries
10501050
runs-on: ubuntu-24.04
1051-
timeout-minutes: 40
1051+
timeout-minutes: 45
10521052
strategy:
10531053
fail-fast: false
10541054
matrix:

go.mod

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -330,7 +330,7 @@ require (
330330
go.yaml.in/yaml/v3 v3.0.4 // indirect
331331
golang.org/x/arch v0.21.0 // indirect
332332
golang.org/x/crypto v0.42.0 // indirect
333-
golang.org/x/exp v0.0.0-20250819193227-8b4c13bb791b // indirect
333+
golang.org/x/exp v0.0.0-20250911091902-df9299821621 // indirect
334334
golang.org/x/oauth2 v0.31.0 // indirect
335335
golang.org/x/sync v0.17.0 // indirect
336336
golang.org/x/term v0.35.0 // indirect

go.sum

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1319,8 +1319,8 @@ golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u0
13191319
golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
13201320
golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
13211321
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY=
1322-
golang.org/x/exp v0.0.0-20250819193227-8b4c13bb791b h1:DXr+pvt3nC887026GRP39Ej11UATqWDmWuS99x26cD0=
1323-
golang.org/x/exp v0.0.0-20250819193227-8b4c13bb791b/go.mod h1:4QTo5u+SEIbbKW1RacMZq1YEfOBqeXa19JeshGi+zc4=
1322+
golang.org/x/exp v0.0.0-20250911091902-df9299821621 h1:2id6c1/gto0kaHYyrixvknJ8tUK/Qs5IsmBtrc+FtgU=
1323+
golang.org/x/exp v0.0.0-20250911091902-df9299821621/go.mod h1:TwQYMMnGpvZyc+JpB/UAuTNIsVJifOlSkrZkhcvpVUk=
13241324
golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs=
13251325
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
13261326
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=

makefiles/e2e.mk

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -135,7 +135,7 @@ kube-ovn-conformance-e2e:
135135
E2E_BRANCH=$(E2E_BRANCH) \
136136
E2E_IP_FAMILY=$(E2E_IP_FAMILY) \
137137
E2E_NETWORK_MODE=$(E2E_NETWORK_MODE) \
138-
ginkgo $(GINKGO_OUTPUT_OPT) $(GINKGO_PARALLEL_OPT) --randomize-all -v --timeout=30m \
138+
ginkgo $(GINKGO_OUTPUT_OPT) $(GINKGO_PARALLEL_OPT) --randomize-all -v --timeout=35m \
139139
--focus=CNI:Kube-OVN ./test/e2e/kube-ovn/kube-ovn.test -- $(TEST_BIN_ARGS)
140140

141141
.PHONY: kube-ovn-ic-conformance-e2e

pkg/controller/pod.go

Lines changed: 37 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1932,6 +1932,8 @@ func (c *Controller) acquireAddress(pod *v1.Pod, podNet *kubeovnNet) (string, st
19321932
macPointer = ptr.To("")
19331933
}
19341934

1935+
var err error
1936+
var nsNets []*kubeovnNet
19351937
ippoolStr := pod.Annotations[fmt.Sprintf(util.IPPoolAnnotationTemplate, podNet.ProviderName)]
19361938
if ippoolStr == "" {
19371939
ns, err := c.namespacesLister.Get(pod.Namespace)
@@ -1942,6 +1944,15 @@ func (c *Controller) acquireAddress(pod *v1.Pod, podNet *kubeovnNet) (string, st
19421944

19431945
if len(ns.Annotations) != 0 {
19441946
if ipPoolList, ok := ns.Annotations[util.IPPoolAnnotation]; ok {
1947+
if nsNets, err = c.getNsAvailableSubnets(pod, podNet); err != nil {
1948+
klog.Errorf("failed to get available subnets for pod %s/%s, %v", pod.Namespace, pod.Name, err)
1949+
return "", "", "", podNet.Subnet, err
1950+
}
1951+
subnetNames := make([]string, 0, len(nsNets))
1952+
for _, net := range nsNets {
1953+
subnetNames = append(subnetNames, net.Subnet.Name)
1954+
}
1955+
19451956
for ipPoolName := range strings.SplitSeq(ipPoolList, ",") {
19461957
ippool, err := c.ippoolLister.Get(ipPoolName)
19471958
if err != nil {
@@ -1965,10 +1976,18 @@ func (c *Controller) acquireAddress(pod *v1.Pod, podNet *kubeovnNet) (string, st
19651976
}
19661977
}
19671978

1968-
if ippool.Spec.Subnet == podNet.Subnet.Name {
1969-
ippoolStr = ippool.Name
1970-
break
1979+
for _, net := range nsNets {
1980+
if net.Subnet.Name == ippool.Spec.Subnet {
1981+
ippoolStr = ippool.Name
1982+
podNet.Subnet = net.Subnet
1983+
break
1984+
}
19711985
}
1986+
break
1987+
}
1988+
if ippoolStr == "" {
1989+
klog.Infof("no available ippool in subnets %s for pod %s/%s", strings.Join(subnetNames, ","), pod.Namespace, pod.Name)
1990+
return "", "", "", podNet.Subnet, ipam.ErrNoAvailable
19721991
}
19731992
}
19741993
}
@@ -2007,9 +2026,14 @@ func (c *Controller) acquireAddress(pod *v1.Pod, podNet *kubeovnNet) (string, st
20072026
portName := ovs.PodNameToPortName(podName, pod.Namespace, podNet.ProviderName)
20082027

20092028
// The static ip can be assigned from any subnet after ns supports multi subnets
2010-
nsNets, _ := c.getNsAvailableSubnets(pod, podNet)
2029+
if nsNets == nil {
2030+
if nsNets, err = c.getNsAvailableSubnets(pod, podNet); err != nil {
2031+
klog.Errorf("failed to get available subnets for pod %s/%s, %v", pod.Namespace, pod.Name, err)
2032+
return "", "", "", podNet.Subnet, err
2033+
}
2034+
}
2035+
20112036
var v4IP, v6IP, mac string
2012-
var err error
20132037

20142038
// Static allocate
20152039
if pod.Annotations[fmt.Sprintf(util.IPAddressAnnotationTemplate, podNet.ProviderName)] != "" {
@@ -2041,9 +2065,13 @@ func (c *Controller) acquireAddress(pod *v1.Pod, podNet *kubeovnNet) (string, st
20412065

20422066
if len(ipPool) == 1 && (!strings.ContainsRune(ipPool[0], ',') && net.ParseIP(ipPool[0]) == nil) {
20432067
var skippedAddrs []string
2068+
pool, err := c.ippoolLister.Get(ipPool[0])
2069+
if err != nil {
2070+
klog.Errorf("failed to get ippool %s: %v", ipPool[0], err)
2071+
return "", "", "", podNet.Subnet, err
2072+
}
20442073
for {
2045-
portName := ovs.PodNameToPortName(podName, pod.Namespace, podNet.ProviderName)
2046-
ipv4, ipv6, mac, err := c.ipam.GetRandomAddress(key, portName, macPointer, podNet.Subnet.Name, ipPool[0], skippedAddrs, !podNet.AllowLiveMigration)
2074+
ipv4, ipv6, mac, err := c.ipam.GetRandomAddress(key, portName, macPointer, pool.Spec.Subnet, ipPool[0], skippedAddrs, !podNet.AllowLiveMigration)
20472075
if err != nil {
20482076
klog.Error(err)
20492077
return "", "", "", podNet.Subnet, err
@@ -2293,17 +2321,16 @@ func (c *Controller) getNameByPod(pod *v1.Pod) string {
22932321

22942322
// When subnet's v4availableIPs is 0 but still there's available ip in exclude-ips, the static ip in exclude-ips can be allocated normal.
22952323
func (c *Controller) getNsAvailableSubnets(pod *v1.Pod, podNet *kubeovnNet) ([]*kubeovnNet, error) {
2296-
var result []*kubeovnNet
22972324
// keep the annotation subnet of the pod in first position
2298-
result = append(result, podNet)
2325+
result := []*kubeovnNet{podNet}
22992326

23002327
ns, err := c.namespacesLister.Get(pod.Namespace)
23012328
if err != nil {
23022329
klog.Errorf("failed to get namespace %s, %v", pod.Namespace, err)
23032330
return nil, err
23042331
}
23052332
if ns.Annotations == nil {
2306-
return nil, nil
2333+
return []*kubeovnNet{}, nil
23072334
}
23082335

23092336
subnetNames := ns.Annotations[util.LogicalSwitchAnnotation]

test/e2e/kube-ovn/ipam/ipam.go

Lines changed: 54 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -573,7 +573,7 @@ var _ = framework.Describe("[group:ipam]", func() {
573573
testSubnet := framework.MakeSubnet(subnetName2, "", testCidr, "", "", "", nil, nil, []string{namespaceName})
574574
subnetClient.CreateSync(testSubnet)
575575

576-
ginkgo.By("Creating IPPool resources ")
576+
ginkgo.By("Creating IPPool resources")
577577
ipsRange1 := framework.RandomIPPool(cidr, ipsCount)
578578
ipsRange2 := framework.RandomIPPool(testCidr, ipsCount)
579579
ippool1 := framework.MakeIPPool(ippoolName, subnetName, ipsRange1, []string{namespaceName})
@@ -607,7 +607,7 @@ var _ = framework.Describe("[group:ipam]", func() {
607607
replicas := 1
608608
ipsCount := 1
609609

610-
ginkgo.By("Creating IPPool resources ")
610+
ginkgo.By("Creating IPPool resources")
611611
ipsRange := framework.RandomIPPool(cidr, ipsCount*2)
612612
ipv4Range, ipv6Range := util.SplitIpsByProtocol(ipsRange)
613613
var ipsRange1, ipsRange2 []string
@@ -667,4 +667,56 @@ var _ = framework.Describe("[group:ipam]", func() {
667667
}
668668
}
669669
})
670+
671+
framework.ConformanceIt("should block IP allocation if the ippool bound by namespace annotation has no available IPs", func() {
672+
f.SkipVersionPriorTo(1, 14, "This feature was introduced in v1.14")
673+
674+
ginkgo.By("Creating IPPool " + ippoolName)
675+
ipsCount := 1
676+
ips := framework.RandomIPPool(cidr, ipsCount)
677+
ippool := framework.MakeIPPool(ippoolName, subnetName, ips, []string{namespaceName})
678+
_ = ippoolClient.CreateSync(ippool)
679+
680+
ginkgo.By("Creating deployment " + deployName + " with replicas equal to the number of IPs in the ippool")
681+
labels := map[string]string{"app": deployName}
682+
deploy := framework.MakeDeployment(deployName, int32(ipsCount), labels, nil, "pause", framework.PauseImage, "")
683+
_ = deployClient.CreateSync(deploy)
684+
685+
ginkgo.By("Creating pod " + podName + " which should be blocked for IP allocation")
686+
pod := framework.MakePod(namespaceName, podName, nil, nil, "", nil, nil)
687+
_ = podClient.Create(pod)
688+
689+
ginkgo.By("Waiting for pod " + podName + " to have event indicating IP allocation failure")
690+
eventClient := f.EventClient()
691+
_ = eventClient.WaitToHaveEvent("Pod", podName, "Warning", "AcquireAddressFailed", "kube-ovn-controller", "")
692+
})
693+
694+
framework.ConformanceIt("should be able to allocate IP from IPPools in different subnets", func() {
695+
f.SkipVersionPriorTo(1, 14, "This feature was introduced in v1.14")
696+
ipsCount := 1
697+
698+
ginkgo.By("Creating subnet " + subnetName2)
699+
cidr2 := framework.RandomCIDR(f.ClusterIPFamily)
700+
subnet2 := framework.MakeSubnet(subnetName2, "", cidr2, "", "", "", nil, nil, []string{namespaceName})
701+
_ = subnetClient.CreateSync(subnet2)
702+
703+
ginkgo.By("Creating IPPool " + ippoolName)
704+
ips := framework.RandomIPPool(cidr, ipsCount)
705+
ippool := framework.MakeIPPool(ippoolName, subnetName, ips, []string{namespaceName})
706+
_ = ippoolClient.CreateSync(ippool)
707+
708+
ginkgo.By("Creating IPPool " + ippoolName2)
709+
ips2 := framework.RandomIPPool(cidr2, ipsCount)
710+
ippool2 := framework.MakeIPPool(ippoolName2, subnetName2, ips2, []string{namespaceName})
711+
_ = ippoolClient.CreateSync(ippool2)
712+
713+
ginkgo.By("Creating deployment " + deployName + " with replicas equal to the number of IPs in the ippool " + ippoolName)
714+
labels := map[string]string{"app": deployName}
715+
deploy := framework.MakeDeployment(deployName, int32(ipsCount), labels, nil, "pause", framework.PauseImage, "")
716+
_ = deployClient.CreateSync(deploy)
717+
718+
ginkgo.By("Creating pod " + podName + " which should have IP allocated from ippool " + ippoolName2)
719+
pod := framework.MakePod(namespaceName, podName, nil, nil, "", nil, nil)
720+
_ = podClient.CreateSync(pod)
721+
})
670722
})

0 commit comments

Comments
 (0)