Merge pull request #95 from qiangzii/master

fix route and ip addr renew period for vlan

Author: yongchuanzhou

pkg/allocator/allocator.go

@@ -311,14 +311,18 @@ func (a *Allocator) IPAddrReNew() {
 	defer a.lock.Unlock()
 
 	for _, nic := range a.nics {
-		nicKey := getNicKey(nic.Nic)
-		if nic.isOK() && nic.Nic.VxNet.TunnelType == qcclient.TunnelTypeVlan {
-			// renew ip lease
-			err := networkutils.UpdateLinkIPAddrAndLease(nic.Nic)
-			if err != nil {
-				log.Errorf("update hostNic %s bridge ip addr lease error: %v", nicKey, err)
+		if nic != nil {
+			nicKey := getNicKey(nic.Nic)
+			if nic.isOK() && nic.Nic.VxNet.TunnelType == constants.TunnelTypeVlan {
+				brName := constants.GetHostNicBridgeName(int(nic.Nic.RouteTableNum))
+				// renew ip lease
+				err := networkutils.UpdateLinkIPAddrAndLease(nic.Nic)
+				if err != nil {
+					log.Errorf("renew hostNic %s bridge %s ip addr lease error: %v", nicKey, brName, err)
+				} else {
+					log.Infof("renew hostNic %s bridge %s ip addr lease success!", nicKey, brName)
+				}
 			}
-			log.Infof("update hostNic %s bridge ip addr lease success!", nicKey)
 		}
 	}
 }
@@ -405,7 +409,7 @@ func (a *Allocator) getVxnetMaxNicNum() int {
 func (a *Allocator) run(stopCh <-chan struct{}) {
 	jobTimer := time.NewTicker(time.Duration(a.conf.Sync) * time.Second).C
 	freeTimer := time.NewTicker(time.Duration(a.conf.FreePeriod) * time.Minute).C
-	ipAddrReNewTimer := time.NewTicker(time.Duration(1) * time.Hour).C
+
 	for {
 		select {
 		case <-stopCh:
@@ -417,7 +421,7 @@ func (a *Allocator) run(stopCh <-chan struct{}) {
 		case <-freeTimer:
 			log.Infof("period free sync")
 			a.ClearFreeHostnic(false)
-		case <-ipAddrReNewTimer:
+		case <-constants.IpAddrReNewTicker.C:
 			log.Infof("ip addr renew sync")
 			a.IPAddrReNew()
 		}

pkg/constants/types.go

@@ -22,6 +22,7 @@ import (
 	"errors"
 	"fmt"
 	"strings"
+	"time"
 
 	"github.com/containernetworking/cni/pkg/types"
 	"github.com/yunify/hostnic-cni/pkg/rpc"
@@ -88,6 +89,11 @@ const (
 	EventDelete = "delete"
 
 	MetricsDummyNamespaceForSubnet = "Dummy-ns-for-unmapped-subnets"
+
+	TunnelTypeVlan = "vlan"
+
+	RouteExistsError    = "file exists"
+	RouteNotExistsError = "no such process"
 )
 
 func GetHostNicBridgeName(routeTableNum int) string {
@@ -149,4 +155,7 @@ type K8sArgs struct {
 var (
 	ErrNoAvailableNIC = errors.New("no free nic")
 	ErrNicNotFound    = errors.New("hostnic not found")
+
+	LastIPAddrRenewPeriod = 60 * 60 * time.Second //s, default 1h
+	IpAddrReNewTicker     = time.NewTicker(LastIPAddrRenewPeriod)
 )

pkg/networkutils/networkutil.go

@@ -6,6 +6,8 @@ import (
 	"net"
 	"os"
 	"os/exec"
+	"strings"
+	"time"
 
 	"github.com/containernetworking/plugins/pkg/ns"
 	"github.com/containernetworking/plugins/pkg/utils/sysctl"
@@ -221,13 +223,14 @@ func (n NetworkUtils) setupBridgeNetwork(link netlink.Link, brName, tunnelType s
 	if err != nil {
 		return fmt.Errorf("failed to set link %s up: %v", la.Name, err)
 	}
-	if tunnelType == qcclient.TunnelTypeVlan {
+	if tunnelType == constants.TunnelTypeVlan {
 		// get an ip addr and add to br
 		// 1.get an ip addr from dhcp server
 		bootConf, err := getIPAddrFromDHCPServer(brName)
 		if err != nil {
 			return fmt.Errorf("failed to get ip address for link %s: %v", brName, err)
 		}
+		klog.Infof("get ip addr %+v success from dhcp server for link %s", bootConf.Addresses, brName)
 
 		// 2. replace addr to br
 		err = replaceLinkIPAddr(br, bootConf.Addresses)
@@ -282,9 +285,21 @@ func (n NetworkUtils) setupRouteTable(nic *rpc.HostNic) error {
 		},
 	}
 
-	for _, r := range routes {
-		if err := netlink.RouteAdd(&r); err != nil && !os.IsExist(err) {
-			return fmt.Errorf("failed to add route %v: %v", r, err)
+	// if tunnel type is vlan, clear route to repair hostnic
+	if nic.VxNet.TunnelType == constants.TunnelTypeVlan {
+		for _, r := range routes {
+			// netlink.RouteDel return error if route not exists: no such process
+			if err := netlink.RouteDel(&r); err != nil && !strings.Contains(err.Error(), constants.RouteNotExistsError) {
+				return fmt.Errorf("failed to del route %v: %v", r, err)
+			}
+		}
+
+	} else {
+		for _, r := range routes {
+			// netlink.RouteAdd return error if route already exists: file exists ; shouldn't use os.IsExist here
+			if err := netlink.RouteAdd(&r); err != nil && !strings.Contains(err.Error(), constants.RouteExistsError) {
+				return fmt.Errorf("failed to add route %v: %v", r, err)
+			}
 		}
 	}
 
@@ -410,7 +425,11 @@ func ExecuteCommand(command string) (string, error) {
 }
 
 func getIPAddrFromDHCPServer(ifname string) (*netboot.BootConf, error) {
-	client := client4.NewClient()
+	// client := client4.NewClient()
+	client := &client4.Client{
+		ReadTimeout:  client4.DefaultReadTimeout * 5,
+		WriteTimeout: client4.DefaultWriteTimeout * 5,
+	}
 	conv, err := client.Exchange(ifname)
 	if err != nil {
 		return nil, fmt.Errorf("dhcp client exchange error: %v", err)
@@ -424,11 +443,19 @@ func getIPAddrFromDHCPServer(ifname string) (*netboot.BootConf, error) {
 }
 
 func replaceLinkIPAddr(link netlink.Link, addres []netboot.AddrConf) error {
+	var shortestLeaseTime time.Duration
 	ifName := link.Attrs().Name
 	if len(addres) < 1 {
 		return fmt.Errorf("there is no avaliable addr for link %s", ifName)
 	}
 	for _, addrConf := range addres {
+		if shortestLeaseTime.Seconds() == 0 {
+			shortestLeaseTime = addrConf.ValidLifetime
+		}
+		if addrConf.ValidLifetime.Seconds() < shortestLeaseTime.Seconds() {
+			shortestLeaseTime = addrConf.ValidLifetime
+		}
+
 		addr := &netlink.Addr{
 			IPNet:       &addrConf.IPNet,
 			ValidLft:    int(addrConf.ValidLifetime.Seconds()),
@@ -439,6 +466,13 @@ func replaceLinkIPAddr(link netlink.Link, addres []netboot.AddrConf) error {
 			return fmt.Errorf("replace addr %+v to link %s error: %v", addr, ifName, err)
 		}
 	}
+
+	if shortestLeaseTime.Seconds()/2 != constants.LastIPAddrRenewPeriod.Seconds() && shortestLeaseTime != 0 {
+		klog.Infof("update LastIPAddrRenewPeriod from %v to %v", constants.LastIPAddrRenewPeriod, shortestLeaseTime/2)
+		constants.LastIPAddrRenewPeriod = shortestLeaseTime / 2
+		constants.IpAddrReNewTicker.Reset(shortestLeaseTime / 2)
+	}
+
 	return nil
 }
 
@@ -450,9 +484,10 @@ func UpdateLinkIPAddrAndLease(nic *rpc.HostNic) error {
 
 	// 1. get an ip form dhcp server
 	bootConf, err := getIPAddrFromDHCPServer(brName)
-	if err != nil {
+	if err != nil || bootConf.Addresses == nil {
 		return fmt.Errorf("failed to get ip address for link %s: %v", brName, err)
 	}
+	klog.Infof("get ip addr %+v success from dhcp server for link %s", bootConf.Addresses, brName)
 
 	// 2. replace addr to br
 	err = replaceLinkIPAddr(br, bootConf.Addresses)

pkg/qcclient/wrapper.go

@@ -28,7 +28,6 @@ const (
 	reservedVIPCount              = 12
 	reservedVIPCountForVlan int64 = 7 //reserve 1/7 ip for hostnic br
 
-	TunnelTypeVlan = "vlan"
 )
 
 var (
@@ -478,7 +477,7 @@ func (q *qingcloudAPIWrapper) getVxNets(ids []string, public bool) ([]*rpc.VxNet
 			if qcVxNet.TunnelType != nil {
 				vxnetItem.TunnelType = *qcVxNet.TunnelType
 			}
-			if vxnetItem.TunnelType == TunnelTypeVlan {
+			if vxnetItem.TunnelType == constants.TunnelTypeVlan {
 				// parse ip_network to get mask; if mask is 24, reserve more ip than specifc in config
 				_, ipNet, err := net.ParseCIDR(vxnetItem.Network)
 				if err != nil {