Merge pull request #97 from qiangzii/master

support custom reserved ip count for vlan

Author: yongchuanzhou

Makefile

@@ -73,7 +73,7 @@ deploy:
 	kustomize build config/${TARGET} > ${DEPLOY}
 
 publish: build tools
-	hack/docker_build.sh ${TAG}
+	hack/docker_build.sh ${REPO} ${TAG}
 
 generate-prototype: 
 	protoc --go_out=. pkg/rpc/message.proto

cmd/tools/vxnet-client/client.go

@@ -10,7 +10,7 @@ import (
 )
 
 func showVxNetInfo(vxnet string) *rpc.VxNet {
-	if vxnets, err := qcclient.QClient.GetVxNets([]string{vxnet}); err != nil {
+	if vxnets, err := qcclient.QClient.GetVxNets([]string{vxnet}, 0); err != nil {
 		fmt.Printf("Get info for vxnet %s failed: %v\n", vxnet, err)
 	} else {
 		if len(vxnets) == 0 {

deploy/hostnic.yaml

@@ -181,6 +181,10 @@ spec:
                   description: The block size to use for IP address assignments from
                     this pool. Defaults to 26 for IPv4 and 112 for IPv6.
                   type: integer
+                customReservedIPCount:
+                  description: CustomReservedIPCount used to specify user custom reserved ip count. Defaults to 0
+                  type: integer
+                  format: int64
                 cidr:
                   description: The pool CIDR.
                   type: string
@@ -320,6 +324,9 @@ spec:
                   description: The block size to use for IP address assignments from
                     this pool. Defaults to 26 for IPv4 and 112 for IPv6.
                   type: integer
+                customReservedIPCount:
+                  description: CustomReservedIPCount used to specify user custom reserved ip count. Defaults to 0
+                  type: integer
                 vxnets:
                   description: vxnets in VxNetPool
                   items:
@@ -474,7 +481,7 @@ spec:
                 fieldRef:
                   apiVersion: v1
                   fieldPath: metadata.namespace
-          image: qingcloud/hostnic-plus:v1.0.7
+          image: qingcloud/hostnic-plus:v1.0.14
           imagePullPolicy: IfNotPresent
           name: hostnic-node
           ports:
@@ -513,7 +520,7 @@ spec:
             - /app/install_hostnic.sh
           command:
             - /bin/sh
-          image: qingcloud/hostnic-plus:v1.0.7
+          image: qingcloud/hostnic-plus:v1.0.14
           imagePullPolicy: IfNotPresent
           name: hostnic-init
           resources: {}
@@ -558,11 +565,11 @@ spec:
             name: hostnic-cfg-cm
           name: hostnic-cfg
         - name: apiaccesskey
-          secret:
+          configMap:
             items:
-              - key: config.yaml
-                path: config.yaml
-            secretName: qcsecret
+            - key: config.yaml
+              path: config.yaml
+            name: hostnic-qingcloud
         - hostPath:
             path: /etc/qingcloud
           name: qingcloud-cfg
@@ -588,7 +595,7 @@ spec:
     spec:
       containers:
         - name: hostnic-controller
-          image: qingcloud/hostnic-plus:v1.0.7
+          image: qingcloud/hostnic-plus:v1.0.14
           command:
             - /app/hostnic-controller
             - --v=5
@@ -608,12 +615,6 @@ spec:
       serviceAccountName: hostnic-node
       hostNetwork: true
       volumes:
-        - name: apiaccesskey
-          secret:
-            items:
-              - key: config.yaml
-                path: config.yaml
-            secretName: qcsecret
         - hostPath:
             path: /etc/qingcloud
           name: qingcloud-cfg
@@ -623,4 +624,10 @@ spec:
         - configMap:
             name: clusterconfig
           name: clusterconfig
+        - configMap:
+            items:
+            - key: config.yaml
+              path: config.yaml
+            name: hostnic-qingcloud
+          name: apiaccesskey
 

hack/docker_build.sh

@@ -4,8 +4,9 @@ set -ex
 set -o pipefail
 
 # push to qingcloud with default latest tag
+REPO="$1"
 REPO=${REPO:-qingcloud}
-TAG="$1"
+TAG="$2"
 
 # hostnic
 docker build -f build/hostnic/Dockerfile -t $REPO/hostnic-plus:$TAG .

pkg/allocator/allocator.go

@@ -163,7 +163,7 @@ func (a *Allocator) getVxnets(vxnet string) (*rpc.VxNet, error) {
 		}
 	}
 
-	result, err := qcclient.QClient.GetVxNets([]string{vxnet})
+	result, err := qcclient.QClient.GetVxNets([]string{vxnet}, 0)
 	if err != nil {
 		return nil, err
 	}

pkg/apis/network/v1alpha1/ippool_types.go

@@ -106,6 +106,10 @@ type IPPoolSpec struct {
 	// The block size to use for IP address assignments from this pool. Defaults to 26 for IPv4 and 112 for IPv6.
 	BlockSize int `json:"blockSize,omitempty"`
 
+	// CustomReservedIPCount used to specify user custom reserved ip count. Defaults to 0
+	// +optional
+	CustomReservedIPCount int64 `json:"customReservedIPCount,omitempty"`
+
 	VLAN VLANConfig `json:"vlanConfig,omitempty"`
 
 	Gateway string  `json:"gateway,omitempty"`

pkg/apis/network/v1alpha1/vxnetpool_types.go

@@ -48,6 +48,10 @@ type VxNetPoolSpec struct {
 
 	// The block size to use for IP address assignments from this pool. Defaults to 26 for IPv4 and 112 for IPv6.
 	BlockSize int `json:"blockSize"`
+
+	// CustomReservedIPCount used to specify user custom reserved ip count. Defaults to 0
+	// +optional
+	CustomReservedIPCount int64 `json:"customReservedIPCount,omitempty"`
 }
 
 type PoolInfo struct {

pkg/controller/vxnet.go

@@ -439,7 +439,7 @@ func (c *VxNetPoolController) handleObject(obj interface{}) {
 // newIPPool creates a new IPPool for a vxnet resource. It also sets
 // the appropriate OwnerReferences on the resource so handleObject can discover
 // the vxnet resource that 'owns' it.
-func (c *VxNetPoolController) createIPPool(name string, blockSize int, vxnet *rpc.VxNet) (*networkv1alpha1.IPPool, error) {
+func (c *VxNetPoolController) createIPPool(name string, blockSize int, customReservedIPCount int64, vxnet *rpc.VxNet) (*networkv1alpha1.IPPool, error) {
 	ippool := &networkv1alpha1.IPPool{
 		ObjectMeta: metav1.ObjectMeta{
 			Name: name,
@@ -448,12 +448,13 @@ func (c *VxNetPoolController) createIPPool(name string, blockSize int, vxnet *rp
 			},
 		},
 		Spec: networkv1alpha1.IPPoolSpec{
-			Type:       networkv1alpha1.IPPoolTypeLocal,
-			CIDR:       vxnet.Network,
-			Gateway:    vxnet.Gateway,
-			BlockSize:  blockSize,
-			RangeStart: vxnet.IPStart,
-			RangeEnd:   vxnet.IPEnd,
+			Type:                  networkv1alpha1.IPPoolTypeLocal,
+			CIDR:                  vxnet.Network,
+			Gateway:               vxnet.Gateway,
+			BlockSize:             blockSize,
+			CustomReservedIPCount: customReservedIPCount,
+			RangeStart:            vxnet.IPStart,
+			RangeEnd:              vxnet.IPEnd,
 		},
 	}
 	return c.clientset.NetworkV1alpha1().IPPools().Create(context.TODO(), ippool, metav1.CreateOptions{})
@@ -535,7 +536,7 @@ func (c *VxNetPoolController) prepareK8SResource(pool *networkv1alpha1.VxNetPool
 			if errors.IsNotFound(err) {
 				// create ippool
 				if v, ok := c.getVxNetInfo(vxnet.Name); ok {
-					if _, err := c.createIPPool(vxnet.Name, pool.Spec.BlockSize, v); err != nil {
+					if _, err := c.createIPPool(vxnet.Name, pool.Spec.BlockSize, pool.Spec.CustomReservedIPCount, v); err != nil {
 						return false, err
 					} else {
 						// handle it's block at next event
@@ -623,7 +624,7 @@ func (c *VxNetPoolController) setJob(id, job string) {
 }
 
 func (c *VxNetPoolController) deleteVIPsByVxnetID(vxnetID string) {
-	vxnets, err := qcclient.QClient.GetVxNets([]string{vxnetID})
+	vxnets, err := qcclient.QClient.GetVxNets([]string{vxnetID}, 0)
 	if err != nil {
 		klog.Errorf("Get info for vxnet %s failed: %v\n", vxnetID, err)
 		return
@@ -681,7 +682,7 @@ func (c *VxNetPoolController) qingCloudSync() {
 		}
 	}
 	if len(needUpdate) > 0 {
-		if result, err := qcclient.QClient.GetVxNets(needUpdate); err != nil {
+		if result, err := qcclient.QClient.GetVxNets(needUpdate, pool.Spec.CustomReservedIPCount); err != nil {
 			klog.Errorf("Get vxnet %v from QingCloud failed: %v", needUpdate, err)
 			return
 		} else {

pkg/qcclient/interface.go

@@ -13,7 +13,7 @@ type QingCloudAPI interface {
 	GetCreatedNicsByName(name string) ([]*rpc.HostNic, error)
 
 	//vxnet info
-	GetVxNets([]string) (map[string]*rpc.VxNet, error)
+	GetVxNets(ids []string, customReservedIPCount int64) (map[string]*rpc.VxNet, error)
 
 	//job info
 	DescribeNicJobs(ids []string) ([]string, map[string]bool, error)

pkg/qcclient/wrapper.go

@@ -4,7 +4,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"math/big"
-	"net"
 	"os"
 	"time"
 
@@ -25,14 +24,14 @@ const (
 	defaultOpTimeout    = 300 * time.Second
 	defaultWaitInterval = 5 * time.Second
 
-	reservedVIPCount              = 12
-	reservedVIPCountForVlan int64 = 7 //reserve 1/7 ip for hostnic br
+	reservedVIPCountForVxlan int64 = 12
+	reservedVIPCountForVlan  int64 = 7 //reserve 1/7 ip for hostnic br
 
 )
 
-var (
-	customReservedIPCount int64 = 0
-)
+// var (
+// 	customReservedIPCount int64 = 0
+// )
 
 type Options struct {
 	Tag string
@@ -177,7 +176,7 @@ func (q *qingcloudAPIWrapper) GetCreatedNicsByName(name string) ([]*rpc.HostNic,
 
 	if len(netIDs) > 0 {
 		tmp := removeDupByMap(netIDs)
-		vxnets, err := q.GetVxNets(tmp)
+		vxnets, err := q.GetVxNets(tmp, 0)
 		if err != nil {
 			return nil, err
 		}
@@ -217,7 +216,7 @@ func (q *qingcloudAPIWrapper) GetCreatedNicsByVxNet(vxnet string) ([]*rpc.HostNi
 
 	if len(netIDs) > 0 {
 		tmp := removeDupByMap(netIDs)
-		vxnets, err := q.GetVxNets(tmp)
+		vxnets, err := q.GetVxNets(tmp, 0)
 		if err != nil {
 			return nil, err
 		}
@@ -446,7 +445,7 @@ func (q *qingcloudAPIWrapper) DescribeNicJobs(ids []string) ([]string, map[strin
 	return left, working, nil
 }
 
-func (q *qingcloudAPIWrapper) getVxNets(ids []string, public bool) ([]*rpc.VxNet, error) {
+func (q *qingcloudAPIWrapper) getVxNets(ids []string, public bool, customReservedIPCount int64) ([]*rpc.VxNet, error) {
 	input := &service.DescribeVxNetsInput{
 		VxNets: service.StringSlice(ids),
 		Limit:  service.Int(constants.NicNumLimit),
@@ -477,21 +476,23 @@ func (q *qingcloudAPIWrapper) getVxNets(ids []string, public bool) ([]*rpc.VxNet
 			if qcVxNet.TunnelType != nil {
 				vxnetItem.TunnelType = *qcVxNet.TunnelType
 			}
+			var reservedIPCount int64
 			if vxnetItem.TunnelType == constants.TunnelTypeVlan {
-				// parse ip_network to get mask; if mask is 24, reserve more ip than specifc in config
-				_, ipNet, err := net.ParseCIDR(vxnetItem.Network)
-				if err != nil {
-					log.Errorf("parse ip_network to get mask error: %v", err)
-				}
-				maskLength, _ := ipNet.Mask.Size()
-				if maskLength == 24 {
-					log.Infof("vxnet %s ip network %s mask is 24, reserve another 64 ip", *qcVxNet.VxNetID, vxnetItem.Network)
-					customReservedIPCount = 64
+				totalCount := IPRangeCount(*qcVxNet.Router.DYNIPStart, *qcVxNet.Router.DYNIPEnd)
+				reservedIPCount = totalCount / reservedVIPCountForVlan
+				maxCustomReservedCount := totalCount - reservedIPCount - 1
+
+				if (customReservedIPCount >= 0 && maxCustomReservedCount < customReservedIPCount) || customReservedIPCount < 0 {
+					return nil, fmt.Errorf("invalid customReservedIPCount %d config in vxnetpool spec, should in range 0~%d", customReservedIPCount, maxCustomReservedCount)
 				}
-				vxnetItem.IPEnd = getIPEndAfterReservedForVlan(*qcVxNet.Router.DYNIPStart, *qcVxNet.Router.DYNIPEnd, reservedVIPCountForVlan, customReservedIPCount)
+				log.Infof("vxnet %s custom reserve ip count: %d", *qcVxNet.VxNetID, customReservedIPCount)
+
 			} else {
-				vxnetItem.IPEnd = getIPEndAfterReserved(*qcVxNet.Router.DYNIPEnd, reservedVIPCount, customReservedIPCount)
+				reservedIPCount = reservedVIPCountForVxlan
+				customReservedIPCount = 0
 			}
+			vxnetItem.IPEnd = getIPEndAfterReserved(*qcVxNet.Router.DYNIPEnd, reservedIPCount, customReservedIPCount)
+
 		} else {
 			return nil, fmt.Errorf("vxnet %s should bind to vpc", *qcVxNet.VxNetID)
 		}
@@ -502,12 +503,12 @@ func (q *qingcloudAPIWrapper) getVxNets(ids []string, public bool) ([]*rpc.VxNet
 	return vxNets, nil
 }
 
-func (q *qingcloudAPIWrapper) GetVxNets(ids []string) (map[string]*rpc.VxNet, error) {
+func (q *qingcloudAPIWrapper) GetVxNets(ids []string, customReservedIPCount int64) (map[string]*rpc.VxNet, error) {
 	if len(ids) <= 0 {
 		return nil, errors.WithStack(fmt.Errorf("GetVxNets should not have empty input"))
 	}
 
-	vxnets, err := q.getVxNets(ids, false)
+	vxnets, err := q.getVxNets(ids, false, customReservedIPCount)
 	if err != nil {
 		return nil, err
 	}
@@ -523,7 +524,7 @@ func (q *qingcloudAPIWrapper) GetVxNets(ids []string) (map[string]*rpc.VxNet, er
 		}
 	}
 	if len(left) > 0 {
-		vxnets, err := q.getVxNets(left, true)
+		vxnets, err := q.getVxNets(left, true, customReservedIPCount)
 		if err != nil {
 			return nil, err
 		}
@@ -571,7 +572,7 @@ func (q *qingcloudAPIWrapper) attachNicTag(nics []string) {
 func (q *qingcloudAPIWrapper) CreateVIPs(vxnet *rpc.VxNet) (string, error) {
 	vipName := constants.NicPrefix + vxnet.ID
 	vipRange := fmt.Sprintf("%s-%s", vxnet.IPStart, vxnet.IPEnd)
-	count := IPRangeCount(vxnet.IPStart, vxnet.IPEnd)
+	count := int(IPRangeCount(vxnet.IPStart, vxnet.IPEnd))
 	input := &service.CreateVIPsInput{
 		Count:    &count,
 		VIPName:  &vipName,
@@ -758,23 +759,16 @@ func (q *qingcloudAPIWrapper) DescribeClusterNodes(clusterID string) ([]*rpc.Nod
 	return nodes, nil
 }
 
-func IPRangeCount(from, to string) int {
+func IPRangeCount(from, to string) int64 {
 	startIP := cnet.ParseIP(from)
 	endIP := cnet.ParseIP(to)
 	startInt := cnet.IPToBigInt(*startIP)
 	endInt := cnet.IPToBigInt(*endIP)
-	return int(big.NewInt(0).Sub(endInt, startInt).Int64() + 1)
+	return big.NewInt(0).Sub(endInt, startInt).Int64() + 1
 }
 
 func getIPEndAfterReserved(end string, reservedCount, customReservedCount int64) string {
 	e := cnet.ParseIP(end)
 	i := big.NewInt(0).Sub(cnet.IPToBigInt(*e), big.NewInt(reservedCount+customReservedCount))
 	return cnet.BigIntToIP(i).String()
 }
-
-func getIPEndAfterReservedForVlan(start, end string, reservedCount, customReservedCount int64) string {
-	s := cnet.ParseIP(start)
-	e := cnet.ParseIP(end)
-	i := big.NewInt(0).Sub(cnet.IPToBigInt(*e), big.NewInt((cnet.IPToBigInt(*e).Int64()-cnet.IPToBigInt(*s).Int64()+1)/reservedCount+customReservedCount))
-	return cnet.BigIntToIP(i).String()
-}