diff --git a/pkg/ipamd/api/v1/types.go b/pkg/ipamd/api/v1/types.go new file mode 100644 index 0000000000..f21ca33910 --- /dev/null +++ b/pkg/ipamd/api/v1/types.go @@ -0,0 +1,265 @@ +package v1 + +import ( + "fmt" + "net" + "time" +) + +const ( + // minENILifeTime is the shortest time before we consider deleting a newly created ENI + minENILifeTime = 1 * time.Minute +) + +// IPAMKey is the IPAM primary key. Quoting CNI spec: +// +// Plugins that store state should do so using a primary key of +// (network name, CNI_CONTAINERID, CNI_IFNAME). +type IPAMKey struct { + NetworkName string `json:"networkName"` + ContainerID string `json:"containerID"` + IfName string `json:"ifName"` +} + +// IsZero returns true if object is equal to the golang zero/null value. +func (k IPAMKey) IsZero() bool { + return k == IPAMKey{} +} + +// String() implements the fmt.Stringer interface. +func (k IPAMKey) String() string { + return fmt.Sprintf("%s/%s/%s", k.NetworkName, k.ContainerID, k.IfName) +} + +// IPAMMetadata is the metadata associated with IP allocations. +type IPAMMetadata struct { + K8SPodNamespace string `json:"k8sPodNamespace,omitempty"` + K8SPodName string `json:"k8sPodName,omitempty"` +} + +// ENI represents a single ENI. Exported fields will be marshaled for introspection. +type ENI struct { + // CreateTime is excluded from marshalled representation, used only by internal datastore + CreateTime time.Time `json:"-"` + + // AWS ENI ID + ID string + // IsPrimary indicates whether ENI is a primary ENI + IsPrimary bool + // IsTrunk indicates whether this ENI is used to provide pods with dedicated ENIs + IsTrunk bool + // IsEFA indicates whether this ENI is tagged as an EFA + IsEFA bool + // DeviceNumber is the device number of ENI (0 means the primary ENI) + DeviceNumber int + // IPv4Addresses shows whether each address is assigned, the key is IP address, which must + // be in dot-decimal notation with no leading zeros and no whitespace(eg: "10.1.0.253") + // Key is the IP address - PD: "IP/28" and SIP: "IP/32" + AvailableIPv4Cidrs map[string]*CidrInfo + //IPv6CIDRs contains information tied to IPv6 Prefixes attached to the ENI + IPv6Cidrs map[string]*CidrInfo +} + +// IsTooYoung returns true if the ENI hasn't been around long enough to be deleted. +func (e *ENI) IsTooYoung() bool { + return time.Since(e.CreateTime) < minENILifeTime +} + +// HasIPInCooling returns true if an IP address was unassigned recently. +func (e *ENI) HasIPInCooling(ipCooldownPeriod time.Duration) bool { + for _, assignedaddr := range e.AvailableIPv4Cidrs { + for _, addr := range assignedaddr.IPAddresses { + if addr.InCoolingPeriod(ipCooldownPeriod) { + return true + } + } + } + return false +} + +// HasPods returns true if the ENI has pods assigned to it. +func (e *ENI) HasPods() bool { + return e.AssignedIPv4Addresses() != 0 +} + +// AddressInfo contains information about an IP, Exported fields will be marshaled for introspection. +type AddressInfo struct { + Address string + + IPAMKey IPAMKey + IPAMMetadata IPAMMetadata + AssignedTime time.Time + UnassignedTime time.Time +} + +// CidrInfo +type CidrInfo struct { + // Either v4/v6 Host or LPM Prefix + Cidr net.IPNet + // Key is individual IP addresses from the Prefix - /32 (v4) or /128 (v6) + IPAddresses map[string]*AddressInfo + // true if Cidr here is an LPM prefix + IsPrefix bool + // IP Address Family of the Cidr + AddressFamily string +} + +func (cidr *CidrInfo) Size() int { + ones, bits := cidr.Cidr.Mask.Size() + return (1 << (bits - ones)) +} + +func (e *ENI) findAddressForSandbox(ipamKey IPAMKey) (*CidrInfo, *AddressInfo) { + // Either v4 or v6 for now. + // Check in V4 prefixes + for _, availableCidr := range e.AvailableIPv4Cidrs { + for _, addr := range availableCidr.IPAddresses { + if addr.IPAMKey == ipamKey { + return availableCidr, addr + } + } + } + + // Check in V6 prefixes + for _, availableCidr := range e.IPv6Cidrs { + for _, addr := range availableCidr.IPAddresses { + if addr.IPAMKey == ipamKey { + return availableCidr, addr + } + } + } + return nil, nil +} + +// AssignedIPv4Addresses is the number of IP addresses already assigned +func (e *ENI) AssignedIPv4Addresses() int { + count := 0 + for _, availableCidr := range e.AvailableIPv4Cidrs { + count += availableCidr.AssignedIPAddressesInCidr() + } + return count +} + +// AssignedIPAddressesInCidr is the number of IP addresses already assigned in the IPv4 CIDR +func (cidr *CidrInfo) AssignedIPAddressesInCidr() int { + count := 0 + //SIP : This will run just once and count will be 0 if addr is not assigned or addr is not allocated yet(unused IP) + //PD : This will return count of number /32 assigned in /28 CIDR. + for _, addr := range cidr.IPAddresses { + if addr.Assigned() { + count++ + } + } + return count +} + +type CidrStats struct { + AssignedIPs int + CooldownIPs int +} + +// Gets number of assigned IPs and the IPs in cooldown from a given CIDR +func (cidr *CidrInfo) GetIPStatsFromCidr(ipCooldownPeriod time.Duration) CidrStats { + stats := CidrStats{} + for _, addr := range cidr.IPAddresses { + if addr.Assigned() { + stats.AssignedIPs++ + } else if addr.InCoolingPeriod(ipCooldownPeriod) { + stats.CooldownIPs++ + } + } + return stats +} + +// Assigned returns true iff the address is allocated to a pod/sandbox. +func (addr AddressInfo) Assigned() bool { + return !addr.IPAMKey.IsZero() +} + +// InCoolingPeriod checks whether an addr is in ipCooldownPeriod +func (addr AddressInfo) InCoolingPeriod(ipCooldownPeriod time.Duration) bool { + return time.Since(addr.UnassignedTime) <= ipCooldownPeriod +} + +// ENIPool is a collection of ENI, keyed by ENI ID +type ENIPool map[string]*ENI + +// AssignedIPv4Addresses is the number of IP addresses already assigned +func (p *ENIPool) AssignedIPv4Addresses() int { + count := 0 + for _, eni := range *p { + count += eni.AssignedIPv4Addresses() + } + return count +} + +// FindAddressForSandbox returns ENI and AddressInfo or (nil, nil) if not found +func (p *ENIPool) FindAddressForSandbox(ipamKey IPAMKey) (*ENI, *CidrInfo, *AddressInfo) { + for _, eni := range *p { + if availableCidr, addr := eni.findAddressForSandbox(ipamKey); addr != nil && availableCidr != nil { + return eni, availableCidr, addr + } + } + return nil, nil, nil +} + +// PodIPInfo contains pod's IP and the device number of the ENI +type PodIPInfo struct { + IPAMKey IPAMKey + // IP is the IPv4 address of pod + IP string + // DeviceNumber is the device number of the ENI + DeviceNumber int +} + +// ENIInfos contains ENI IP information +type ENIInfos struct { + // TotalIPs is the total number of IP addresses + TotalIPs int + // assigned is the number of IP addresses that has been assigned + AssignedIPs int + // ENIs contains ENI IP pool information + ENIs map[string]ENI +} + +// CheckpointFormatVersion is the version stamp used on stored checkpoints. +const CheckpointFormatVersion = "vpc-cni-ipam/1" + +// CheckpointData is the format of stored checkpoints. Note this is +// deliberately a "dumb" format since efficiency is less important +// than version stability here. +type CheckpointData struct { + Version string `json:"version"` + Allocations []CheckpointEntry `json:"allocations"` +} + +// CheckpointEntry is a "row" in the conceptual IPAM datastore, as stored +// in checkpoints. +type CheckpointEntry struct { + IPAMKey + IPv4 string `json:"ipv4,omitempty"` + IPv6 string `json:"ipv6,omitempty"` + AllocationTimestamp int64 `json:"allocationTimestamp"` + Metadata IPAMMetadata `json:"metadata"` +} + +type DataStoreStats struct { + // Total number of addresses allocated + TotalIPs int + // Total number of prefixes allocated + TotalPrefixes int + + // Number of assigned addresses + AssignedIPs int + // Number of addresses in cooldown + CooldownIPs int +} + +func (stats *DataStoreStats) String() string { + return fmt.Sprintf("Total IPs/Prefixes = %d/%d, AssignedIPs/CooldownIPs: %d/%d", + stats.TotalIPs, stats.TotalPrefixes, stats.AssignedIPs, stats.CooldownIPs) +} + +func (stats *DataStoreStats) AvailableAddresses() int { + return stats.TotalIPs - stats.AssignedIPs +} diff --git a/pkg/ipamd/datastore/data_store.go b/pkg/ipamd/datastore/data_store.go index 02d5cd21f0..509d6afad9 100644 --- a/pkg/ipamd/datastore/data_store.go +++ b/pkg/ipamd/datastore/data_store.go @@ -21,6 +21,7 @@ import ( "sync" "time" + v1 "github.com/aws/amazon-vpc-cni-k8s/pkg/ipamd/api/v1" "github.com/aws/amazon-vpc-cni-k8s/pkg/netlinkwrapper" "github.com/aws/amazon-vpc-cni-k8s/pkg/networkutils" "github.com/vishvananda/netlink" @@ -34,9 +35,6 @@ import ( ) const ( - // minENILifeTime is the shortest time before we consider deleting a newly created ENI - minENILifeTime = 1 * time.Minute - // envIPCooldownPeriod (default 30 seconds) specifies the time after pod deletion before an IP can be assigned to a new pod envIPCooldownPeriod = "IP_COOLDOWN_PERIOD" @@ -79,147 +77,6 @@ const backfillNetworkIface = "unknown" // ErrUnknownPod is an error when there is no pod in data store matching pod name, namespace, sandbox id var ErrUnknownPod = errors.New("datastore: unknown pod") -// IPAMKey is the IPAM primary key. Quoting CNI spec: -// -// Plugins that store state should do so using a primary key of -// (network name, CNI_CONTAINERID, CNI_IFNAME). -type IPAMKey struct { - NetworkName string `json:"networkName"` - ContainerID string `json:"containerID"` - IfName string `json:"ifName"` -} - -// IsZero returns true if object is equal to the golang zero/null value. -func (k IPAMKey) IsZero() bool { - return k == IPAMKey{} -} - -// String() implements the fmt.Stringer interface. -func (k IPAMKey) String() string { - return fmt.Sprintf("%s/%s/%s", k.NetworkName, k.ContainerID, k.IfName) -} - -// IPAMMetadata is the metadata associated with IP allocations. -type IPAMMetadata struct { - K8SPodNamespace string `json:"k8sPodNamespace,omitempty"` - K8SPodName string `json:"k8sPodName,omitempty"` -} - -// ENI represents a single ENI. Exported fields will be marshaled for introspection. -type ENI struct { - // AWS ENI ID - ID string - createTime time.Time - // IsPrimary indicates whether ENI is a primary ENI - IsPrimary bool - // IsTrunk indicates whether this ENI is used to provide pods with dedicated ENIs - IsTrunk bool - // IsEFA indicates whether this ENI is tagged as an EFA - IsEFA bool - // DeviceNumber is the device number of ENI (0 means the primary ENI) - DeviceNumber int - // IPv4Addresses shows whether each address is assigned, the key is IP address, which must - // be in dot-decimal notation with no leading zeros and no whitespace(eg: "10.1.0.253") - // Key is the IP address - PD: "IP/28" and SIP: "IP/32" - AvailableIPv4Cidrs map[string]*CidrInfo - //IPv6CIDRs contains information tied to IPv6 Prefixes attached to the ENI - IPv6Cidrs map[string]*CidrInfo -} - -// AddressInfo contains information about an IP, Exported fields will be marshaled for introspection. -type AddressInfo struct { - Address string - - IPAMKey IPAMKey - IPAMMetadata IPAMMetadata - AssignedTime time.Time - UnassignedTime time.Time -} - -// CidrInfo -type CidrInfo struct { - // Either v4/v6 Host or LPM Prefix - Cidr net.IPNet - // Key is individual IP addresses from the Prefix - /32 (v4) or /128 (v6) - IPAddresses map[string]*AddressInfo - // true if Cidr here is an LPM prefix - IsPrefix bool - // IP Address Family of the Cidr - AddressFamily string -} - -func (cidr *CidrInfo) Size() int { - ones, bits := cidr.Cidr.Mask.Size() - return (1 << (bits - ones)) -} - -func (e *ENI) findAddressForSandbox(ipamKey IPAMKey) (*CidrInfo, *AddressInfo) { - // Either v4 or v6 for now. - // Check in V4 prefixes - for _, availableCidr := range e.AvailableIPv4Cidrs { - for _, addr := range availableCidr.IPAddresses { - if addr.IPAMKey == ipamKey { - return availableCidr, addr - } - } - } - - // Check in V6 prefixes - for _, availableCidr := range e.IPv6Cidrs { - for _, addr := range availableCidr.IPAddresses { - if addr.IPAMKey == ipamKey { - return availableCidr, addr - } - } - } - return nil, nil -} - -// AssignedIPv4Addresses is the number of IP addresses already assigned -func (e *ENI) AssignedIPv4Addresses() int { - count := 0 - for _, availableCidr := range e.AvailableIPv4Cidrs { - count += availableCidr.AssignedIPAddressesInCidr() - } - return count -} - -// AssignedIPAddressesInCidr is the number of IP addresses already assigned in the IPv4 CIDR -func (cidr *CidrInfo) AssignedIPAddressesInCidr() int { - count := 0 - //SIP : This will run just once and count will be 0 if addr is not assigned or addr is not allocated yet(unused IP) - //PD : This will return count of number /32 assigned in /28 CIDR. - for _, addr := range cidr.IPAddresses { - if addr.Assigned() { - count++ - } - } - return count -} - -type CidrStats struct { - AssignedIPs int - CooldownIPs int -} - -// Gets number of assigned IPs and the IPs in cooldown from a given CIDR -func (cidr *CidrInfo) GetIPStatsFromCidr(ipCooldownPeriod time.Duration) CidrStats { - stats := CidrStats{} - for _, addr := range cidr.IPAddresses { - if addr.Assigned() { - stats.AssignedIPs++ - } else if addr.inCoolingPeriod(ipCooldownPeriod) { - stats.CooldownIPs++ - } - } - return stats -} - -// Assigned returns true iff the address is allocated to a pod/sandbox. -func (addr AddressInfo) Assigned() bool { - return !addr.IPAMKey.IsZero() -} - // getCooldownPeriod returns the time duration in seconds configured by the IP_COOLDOWN_PERIOD env variable func getCooldownPeriod() time.Duration { cooldownVal, err, _ := utils.GetIntFromStringEnvVar(envIPCooldownPeriod, 30) @@ -229,48 +86,12 @@ func getCooldownPeriod() time.Duration { return time.Duration(cooldownVal) * time.Second } -// InCoolingPeriod checks whether an addr is in ipCooldownPeriod -func (addr AddressInfo) inCoolingPeriod(ipCooldownPeriod time.Duration) bool { - return time.Since(addr.UnassignedTime) <= ipCooldownPeriod -} - -// ENIPool is a collection of ENI, keyed by ENI ID -type ENIPool map[string]*ENI - -// AssignedIPv4Addresses is the number of IP addresses already assigned -func (p *ENIPool) AssignedIPv4Addresses() int { - count := 0 - for _, eni := range *p { - count += eni.AssignedIPv4Addresses() - } - return count -} - -// FindAddressForSandbox returns ENI and AddressInfo or (nil, nil) if not found -func (p *ENIPool) FindAddressForSandbox(ipamKey IPAMKey) (*ENI, *CidrInfo, *AddressInfo) { - for _, eni := range *p { - if availableCidr, addr := eni.findAddressForSandbox(ipamKey); addr != nil && availableCidr != nil { - return eni, availableCidr, addr - } - } - return nil, nil, nil -} - -// PodIPInfo contains pod's IP and the device number of the ENI -type PodIPInfo struct { - IPAMKey IPAMKey - // IP is the IPv4 address of pod - IP string - // DeviceNumber is the device number of the ENI - DeviceNumber int -} - // DataStore contains node level ENI/IP type DataStore struct { total int assigned int allocatedPrefix int - eniPool ENIPool + eniPool v1.ENIPool lock sync.Mutex log logger.Logger backingStore Checkpointer @@ -279,20 +100,10 @@ type DataStore struct { ipCooldownPeriod time.Duration } -// ENIInfos contains ENI IP information -type ENIInfos struct { - // TotalIPs is the total number of IP addresses - TotalIPs int - // assigned is the number of IP addresses that has been assigned - AssignedIPs int - // ENIs contains ENI IP pool information - ENIs map[string]ENI -} - // NewDataStore returns DataStore structure func NewDataStore(log logger.Logger, backingStore Checkpointer, isPDEnabled bool) *DataStore { return &DataStore{ - eniPool: make(ENIPool), + eniPool: make(v1.ENIPool), log: log, backingStore: backingStore, netLink: netlinkwrapper.NewNetLink(), @@ -301,31 +112,10 @@ func NewDataStore(log logger.Logger, backingStore Checkpointer, isPDEnabled bool } } -// CheckpointFormatVersion is the version stamp used on stored checkpoints. -const CheckpointFormatVersion = "vpc-cni-ipam/1" - -// CheckpointData is the format of stored checkpoints. Note this is -// deliberately a "dumb" format since efficiency is less important -// than version stability here. -type CheckpointData struct { - Version string `json:"version"` - Allocations []CheckpointEntry `json:"allocations"` -} - -// CheckpointEntry is a "row" in the conceptual IPAM datastore, as stored -// in checkpoints. -type CheckpointEntry struct { - IPAMKey - IPv4 string `json:"ipv4,omitempty"` - IPv6 string `json:"ipv6,omitempty"` - AllocationTimestamp int64 `json:"allocationTimestamp"` - Metadata IPAMMetadata `json:"metadata"` -} - // ReadBackingStore initializes the IP allocation state from the // configured backing store. Should be called before using data store. func (ds *DataStore) ReadBackingStore(isv6Enabled bool) error { - var data CheckpointData + var data v1.CheckpointData // Read from checkpoint file ds.log.Infof("Begin ipam state recovery from backing store") @@ -339,8 +129,8 @@ func (ds *DataStore) ReadBackingStore(isv6Enabled bool) error { } return errors.Wrap(err, "failed ipam state recovery from backing store") } - if data.Version != CheckpointFormatVersion { - return errors.Errorf("failed ipam state recovery due to unexpected checkpointVersion: %v/%v", data.Version, CheckpointFormatVersion) + if data.Version != v1.CheckpointFormatVersion { + return errors.Errorf("failed ipam state recovery due to unexpected checkpointVersion: %v/%v", data.Version, v1.CheckpointFormatVersion) } if normalizedData, err := ds.normalizeCheckpointDataByPodVethExistence(data); err != nil { return errors.Wrap(err, "failed normalize checkpoint data with veth check") @@ -373,7 +163,7 @@ func (ds *DataStore) ReadBackingStore(isv6Enabled bool) error { if _, ok := cidr.IPAddresses[ipAddr.String()]; ok { return errors.New(IPAlreadyInStoreError) } - addr := &AddressInfo{Address: ipAddr.String()} + addr := &v1.AddressInfo{Address: ipAddr.String()} cidr.IPAddresses[ipAddr.String()] = addr ds.assignPodIPAddressUnsafe(addr, allocation.IPAMKey, allocation.Metadata, time.Unix(0, allocation.AllocationTimestamp)) ds.log.Debugf("Recovered %s => %s/%s", allocation.IPAMKey, eni.ID, addr.Address) @@ -402,14 +192,14 @@ func (ds *DataStore) ReadBackingStore(isv6Enabled bool) error { } func (ds *DataStore) writeBackingStoreUnsafe() error { - allocations := make([]CheckpointEntry, 0, ds.assigned) + allocations := make([]v1.CheckpointEntry, 0, ds.assigned) for _, eni := range ds.eniPool { // Loop through ENI's v4 prefixes for _, assignedAddr := range eni.AvailableIPv4Cidrs { for _, addr := range assignedAddr.IPAddresses { if addr.Assigned() { - entry := CheckpointEntry{ + entry := v1.CheckpointEntry{ IPAMKey: addr.IPAMKey, IPv4: addr.Address, AllocationTimestamp: addr.AssignedTime.UnixNano(), @@ -423,7 +213,7 @@ func (ds *DataStore) writeBackingStoreUnsafe() error { for _, assignedAddr := range eni.IPv6Cidrs { for _, addr := range assignedAddr.IPAddresses { if addr.Assigned() { - entry := CheckpointEntry{ + entry := v1.CheckpointEntry{ IPAMKey: addr.IPAMKey, IPv6: addr.Address, AllocationTimestamp: addr.AssignedTime.UnixNano(), @@ -435,8 +225,8 @@ func (ds *DataStore) writeBackingStoreUnsafe() error { } } - data := CheckpointData{ - Version: CheckpointFormatVersion, + data := v1.CheckpointData{ + Version: v1.CheckpointFormatVersion, Allocations: allocations, } @@ -454,14 +244,14 @@ func (ds *DataStore) AddENI(eniID string, deviceNumber int, isPrimary, isTrunk, if ok { return errors.New(DuplicatedENIError) } - ds.eniPool[eniID] = &ENI{ - createTime: time.Now(), + ds.eniPool[eniID] = &v1.ENI{ + CreateTime: time.Now(), IsPrimary: isPrimary, IsTrunk: isTrunk, IsEFA: isEFA, ID: eniID, DeviceNumber: deviceNumber, - AvailableIPv4Cidrs: make(map[string]*CidrInfo)} + AvailableIPv4Cidrs: make(map[string]*v1.CidrInfo)} prometheusmetrics.Enis.Set(float64(len(ds.eniPool))) // Initialize ENI IPs In Use to 0 when an ENI is created @@ -488,9 +278,9 @@ func (ds *DataStore) AddIPv4CidrToStore(eniID string, ipv4Cidr net.IPNet, isPref return errors.New(IPAlreadyInStoreError) } - newCidrInfo := &CidrInfo{ + newCidrInfo := &v1.CidrInfo{ Cidr: ipv4Cidr, - IPAddresses: make(map[string]*AddressInfo), + IPAddresses: make(map[string]*v1.AddressInfo), IsPrefix: isPrefix, AddressFamily: "4", } @@ -519,7 +309,7 @@ func (ds *DataStore) DelIPv4CidrFromStore(eniID string, cidr net.IPNet, force bo } strIPv4Cidr := cidr.String() - var deletableCidr *CidrInfo + var deletableCidr *v1.CidrInfo deletableCidr, ok = curENI.AvailableIPv4Cidrs[strIPv4Cidr] if !ok { ds.log.Debugf("Unknown %s CIDR", strIPv4Cidr) @@ -582,11 +372,11 @@ func (ds *DataStore) AddIPv6CidrToStore(eniID string, ipv6Cidr net.IPNet, isPref ds.log.Debugf("Assigning IPv6CIDRs") if curENI.IPv6Cidrs == nil { - curENI.IPv6Cidrs = make(map[string]*CidrInfo) + curENI.IPv6Cidrs = make(map[string]*v1.CidrInfo) } - curENI.IPv6Cidrs[strIPv6Cidr] = &CidrInfo{ + curENI.IPv6Cidrs[strIPv6Cidr] = &v1.CidrInfo{ Cidr: ipv6Cidr, - IPAddresses: make(map[string]*AddressInfo), + IPAddresses: make(map[string]*v1.AddressInfo), IsPrefix: isPrefix, AddressFamily: "6", } @@ -600,7 +390,7 @@ func (ds *DataStore) AddIPv6CidrToStore(eniID string, ipv6Cidr net.IPNet, isPref return nil } -func (ds *DataStore) AssignPodIPAddress(ipamKey IPAMKey, ipamMetadata IPAMMetadata, isIPv4Enabled bool, isIPv6Enabled bool) (ipv4Address string, +func (ds *DataStore) AssignPodIPAddress(ipamKey v1.IPAMKey, ipamMetadata v1.IPAMMetadata, isIPv4Enabled bool, isIPv6Enabled bool) (ipv4Address string, ipv6Address string, deviceNumber int, err error) { //Currently it's either v4 or v6. Dual Stack mode isn't supported. if isIPv4Enabled { @@ -612,7 +402,7 @@ func (ds *DataStore) AssignPodIPAddress(ipamKey IPAMKey, ipamMetadata IPAMMetada } // AssignPodIPv6Address assigns an IPv6 address to pod. Returns the assigned IPv6 address along with device number -func (ds *DataStore) AssignPodIPv6Address(ipamKey IPAMKey, ipamMetadata IPAMMetadata) (ipv6Address string, deviceNumber int, err error) { +func (ds *DataStore) AssignPodIPv6Address(ipamKey v1.IPAMKey, ipamMetadata v1.IPAMMetadata) (ipv6Address string, deviceNumber int, err error) { ds.lock.Lock() defer ds.lock.Unlock() @@ -643,7 +433,7 @@ func (ds *DataStore) AssignPodIPv6Address(ipamKey IPAMKey, ipamMetadata IPAMMeta continue } ds.log.Debugf("New v6 IP from PD pool- %s", ipv6Address) - addr := &AddressInfo{Address: ipv6Address} + addr := &v1.AddressInfo{Address: ipv6Address} V6Cidr.IPAddresses[ipv6Address] = addr ds.assignPodIPAddressUnsafe(addr, ipamKey, ipamMetadata, time.Now()) @@ -666,7 +456,7 @@ func (ds *DataStore) AssignPodIPv6Address(ipamKey IPAMKey, ipamMetadata IPAMMeta // AssignPodIPv4Address assigns an IPv4 address to pod // It returns the assigned IPv4 address, device number, error -func (ds *DataStore) AssignPodIPv4Address(ipamKey IPAMKey, ipamMetadata IPAMMetadata) (ipv4address string, deviceNumber int, err error) { +func (ds *DataStore) AssignPodIPv4Address(ipamKey v1.IPAMKey, ipamMetadata v1.IPAMMetadata) (ipv4address string, deviceNumber int, err error) { ds.lock.Lock() defer ds.lock.Unlock() @@ -679,7 +469,7 @@ func (ds *DataStore) AssignPodIPv4Address(ipamKey IPAMKey, ipamMetadata IPAMMeta for _, eni := range ds.eniPool { for _, availableCidr := range eni.AvailableIPv4Cidrs { - var addr *AddressInfo + var addr *v1.AddressInfo var strPrivateIPv4 string var err error @@ -692,7 +482,7 @@ func (ds *DataStore) AssignPodIPv4Address(ipamKey IPAMKey, ipamMetadata IPAMMeta } ds.log.Debugf("New IP from CIDR pool- %s", strPrivateIPv4) if availableCidr.IPAddresses == nil { - availableCidr.IPAddresses = make(map[string]*AddressInfo) + availableCidr.IPAddresses = make(map[string]*v1.AddressInfo) } // Update prometheus for ips per cidr // Secondary IP mode will have /32:1 and Prefix mode will have /28: @@ -707,7 +497,7 @@ func (ds *DataStore) AssignPodIPv4Address(ipamKey IPAMKey, ipamMetadata IPAMMeta if addr == nil { // addr is nil when we are using a new IP from prefix or SIP pool // if addr is out of cooldown or not assigned, we can reuse addr - addr = &AddressInfo{Address: strPrivateIPv4} + addr = &v1.AddressInfo{Address: strPrivateIPv4} } availableCidr.IPAddresses[strPrivateIPv4] = addr @@ -736,7 +526,7 @@ func (ds *DataStore) AssignPodIPv4Address(ipamKey IPAMKey, ipamMetadata IPAMMeta } // assignPodIPAddressUnsafe mark Address as assigned. -func (ds *DataStore) assignPodIPAddressUnsafe(addr *AddressInfo, ipamKey IPAMKey, ipamMetadata IPAMMetadata, assignedTime time.Time) { +func (ds *DataStore) assignPodIPAddressUnsafe(addr *v1.AddressInfo, ipamKey v1.IPAMKey, ipamMetadata v1.IPAMMetadata, assignedTime time.Time) { ds.log.Infof("assignPodIPAddressUnsafe: Assign IP %v to sandbox %s", addr.Address, ipamKey) @@ -753,47 +543,26 @@ func (ds *DataStore) assignPodIPAddressUnsafe(addr *AddressInfo, ipamKey IPAMKey } // unassignPodIPAddressUnsafe mark Address as unassigned. -func (ds *DataStore) unassignPodIPAddressUnsafe(addr *AddressInfo) { +func (ds *DataStore) unassignPodIPAddressUnsafe(addr *v1.AddressInfo) { if !addr.Assigned() { // Already unassigned return } ds.log.Infof("unassignPodIPAddressUnsafe: Unassign IP %v from sandbox %s", addr.Address, addr.IPAMKey) - addr.IPAMKey = IPAMKey{} // unassign the addr - addr.IPAMMetadata = IPAMMetadata{} + addr.IPAMKey = v1.IPAMKey{} // unassign the addr + addr.IPAMMetadata = v1.IPAMMetadata{} ds.assigned-- // Prometheus gauge prometheusmetrics.AssignedIPs.Set(float64(ds.assigned)) } -type DataStoreStats struct { - // Total number of addresses allocated - TotalIPs int - // Total number of prefixes allocated - TotalPrefixes int - - // Number of assigned addresses - AssignedIPs int - // Number of addresses in cooldown - CooldownIPs int -} - -func (stats *DataStoreStats) String() string { - return fmt.Sprintf("Total IPs/Prefixes = %d/%d, AssignedIPs/CooldownIPs: %d/%d", - stats.TotalIPs, stats.TotalPrefixes, stats.AssignedIPs, stats.CooldownIPs) -} - -func (stats *DataStoreStats) AvailableAddresses() int { - return stats.TotalIPs - stats.AssignedIPs -} - // GetIPStats returns DataStoreStats for addressFamily -func (ds *DataStore) GetIPStats(addressFamily string) *DataStoreStats { +func (ds *DataStore) GetIPStats(addressFamily string) *v1.DataStoreStats { ds.lock.Lock() defer ds.lock.Unlock() - stats := &DataStoreStats{ + stats := &v1.DataStoreStats{ TotalPrefixes: ds.allocatedPrefix, } for _, eni := range ds.eniPool { @@ -842,7 +611,7 @@ func (ds *DataStore) GetEFAENIs() map[string]bool { } // IsRequiredForWarmIPTarget determines if this ENI has warm IPs that are required to fulfill whatever WARM_IP_TARGET is set to. -func (ds *DataStore) isRequiredForWarmIPTarget(warmIPTarget int, eni *ENI) bool { +func (ds *DataStore) isRequiredForWarmIPTarget(warmIPTarget int, eni *v1.ENI) bool { otherWarmIPs := 0 for _, other := range ds.eniPool { if other.ID != eni.ID { @@ -863,7 +632,7 @@ func (ds *DataStore) isRequiredForWarmIPTarget(warmIPTarget int, eni *ENI) bool } // IsRequiredForMinimumIPTarget determines if this ENI is necessary to fulfill whatever MINIMUM_IP_TARGET is set to. -func (ds *DataStore) isRequiredForMinimumIPTarget(minimumIPTarget int, eni *ENI) bool { +func (ds *DataStore) isRequiredForMinimumIPTarget(minimumIPTarget int, eni *v1.ENI) bool { otherIPs := 0 for _, other := range ds.eniPool { if other.ID != eni.ID { @@ -884,7 +653,7 @@ func (ds *DataStore) isRequiredForMinimumIPTarget(minimumIPTarget int, eni *ENI) } // IsRequiredForWarmPrefixTarget determines if this ENI is necessary to fulfill whatever WARM_PREFIX_TARGET is set to. -func (ds *DataStore) isRequiredForWarmPrefixTarget(warmPrefixTarget int, eni *ENI) bool { +func (ds *DataStore) isRequiredForWarmPrefixTarget(warmPrefixTarget int, eni *v1.ENI) bool { freePrefixes := 0 for _, other := range ds.eniPool { if other.ID != eni.ID { @@ -898,24 +667,24 @@ func (ds *DataStore) isRequiredForWarmPrefixTarget(warmPrefixTarget int, eni *EN return freePrefixes < warmPrefixTarget } -func (ds *DataStore) getDeletableENI(warmIPTarget, minimumIPTarget, warmPrefixTarget int) *ENI { +func (ds *DataStore) getDeletableENI(warmIPTarget, minimumIPTarget, warmPrefixTarget int) *v1.ENI { for _, eni := range ds.eniPool { if eni.IsPrimary { ds.log.Debugf("ENI %s cannot be deleted because it is primary", eni.ID) continue } - if eni.isTooYoung() { + if eni.IsTooYoung() { ds.log.Debugf("ENI %s cannot be deleted because it is too young", eni.ID) continue } - if eni.hasIPInCooling(ds.ipCooldownPeriod) { + if eni.HasIPInCooling(ds.ipCooldownPeriod) { ds.log.Debugf("ENI %s cannot be deleted because has IPs in cooling", eni.ID) continue } - if eni.hasPods() { + if eni.HasPods() { ds.log.Debugf("ENI %s cannot be deleted because it has pods assigned", eni.ID) continue } @@ -951,31 +720,9 @@ func (ds *DataStore) getDeletableENI(warmIPTarget, minimumIPTarget, warmPrefixTa return nil } -// IsTooYoung returns true if the ENI hasn't been around long enough to be deleted. -func (e *ENI) isTooYoung() bool { - return time.Since(e.createTime) < minENILifeTime -} - -// HasIPInCooling returns true if an IP address was unassigned recently. -func (e *ENI) hasIPInCooling(ipCooldownPeriod time.Duration) bool { - for _, assignedaddr := range e.AvailableIPv4Cidrs { - for _, addr := range assignedaddr.IPAddresses { - if addr.inCoolingPeriod(ipCooldownPeriod) { - return true - } - } - } - return false -} - -// HasPods returns true if the ENI has pods assigned to it. -func (e *ENI) hasPods() bool { - return e.AssignedIPv4Addresses() != 0 -} - // GetAllocatableENIs finds ENIs in the datastore that needs more IP addresses allocated -func (ds *DataStore) GetAllocatableENIs(maxIPperENI int, skipPrimary bool) []*ENI { - var enis []*ENI +func (ds *DataStore) GetAllocatableENIs(maxIPperENI int, skipPrimary bool) []*v1.ENI { + var enis []*v1.ENI ds.lock.Lock() defer ds.lock.Unlock() for _, eni := range ds.eniPool { @@ -1035,7 +782,7 @@ func (ds *DataStore) RemoveENIFromDataStore(eniID string, force bool) error { return errors.New(UnknownENIError) } - if eni.hasPods() { + if eni.HasPods() { if !force { return errors.New(ENIInUseError) } @@ -1082,7 +829,7 @@ func (ds *DataStore) RemoveENIFromDataStore(eniID string, force bool) error { // UnassignPodIPAddress a) find out the IP address based on PodName and PodNameSpace // b) mark IP address as unassigned c) returns IP address, ENI's device number, error -func (ds *DataStore) UnassignPodIPAddress(ipamKey IPAMKey) (e *ENI, ip string, deviceNumber int, err error) { +func (ds *DataStore) UnassignPodIPAddress(ipamKey v1.IPAMKey) (e *v1.ENI, ip string, deviceNumber int, err error) { ds.lock.Lock() defer ds.lock.Unlock() ds.log.Debugf("UnassignPodIPAddress: IP address pool stats: total %d, assigned %d, sandbox %s", ds.total, ds.assigned, ipamKey) @@ -1124,16 +871,16 @@ func (ds *DataStore) UnassignPodIPAddress(ipamKey IPAMKey) (e *ENI, ip string, d // AllocatedIPs returns a recent snapshot of allocated sandbox<->IPs. // Note result may already be stale by the time you look at it. -func (ds *DataStore) AllocatedIPs() []PodIPInfo { +func (ds *DataStore) AllocatedIPs() []v1.PodIPInfo { ds.lock.Lock() defer ds.lock.Unlock() - ret := make([]PodIPInfo, 0, ds.eniPool.AssignedIPv4Addresses()) + ret := make([]v1.PodIPInfo, 0, ds.eniPool.AssignedIPv4Addresses()) for _, eni := range ds.eniPool { for _, assignedaddr := range eni.AvailableIPv4Cidrs { for _, addr := range assignedaddr.IPAddresses { if addr.Assigned() { - info := PodIPInfo{ + info := v1.PodIPInfo{ IPAMKey: addr.IPAMKey, IP: addr.Address, DeviceNumber: eni.DeviceNumber, @@ -1190,24 +937,24 @@ func (ds *DataStore) FreeablePrefixes(eniID string) []net.IPNet { } // GetENIInfos provides ENI and IP information about the datastore -func (ds *DataStore) GetENIInfos() *ENIInfos { +func (ds *DataStore) GetENIInfos() *v1.ENIInfos { ds.lock.Lock() defer ds.lock.Unlock() - var eniInfos = ENIInfos{ + var eniInfos = v1.ENIInfos{ TotalIPs: ds.total, AssignedIPs: ds.assigned, - ENIs: make(map[string]ENI, len(ds.eniPool)), + ENIs: make(map[string]v1.ENI, len(ds.eniPool)), } for eni, eniInfo := range ds.eniPool { tmpENIInfo := *eniInfo - tmpENIInfo.AvailableIPv4Cidrs = make(map[string]*CidrInfo, len(eniInfo.AvailableIPv4Cidrs)) - tmpENIInfo.IPv6Cidrs = make(map[string]*CidrInfo, len(eniInfo.IPv6Cidrs)) + tmpENIInfo.AvailableIPv4Cidrs = make(map[string]*v1.CidrInfo, len(eniInfo.AvailableIPv4Cidrs)) + tmpENIInfo.IPv6Cidrs = make(map[string]*v1.CidrInfo, len(eniInfo.IPv6Cidrs)) for cidr := range eniInfo.AvailableIPv4Cidrs { - tmpENIInfo.AvailableIPv4Cidrs[cidr] = &CidrInfo{ + tmpENIInfo.AvailableIPv4Cidrs[cidr] = &v1.CidrInfo{ Cidr: eniInfo.AvailableIPv4Cidrs[cidr].Cidr, - IPAddresses: make(map[string]*AddressInfo, len(eniInfo.AvailableIPv4Cidrs[cidr].IPAddresses)), + IPAddresses: make(map[string]*v1.AddressInfo, len(eniInfo.AvailableIPv4Cidrs[cidr].IPAddresses)), IsPrefix: eniInfo.AvailableIPv4Cidrs[cidr].IsPrefix, } // Since IP Addresses might get removed, we need to make a deep copy here. @@ -1217,9 +964,9 @@ func (ds *DataStore) GetENIInfos() *ENIInfos { } } for cidr := range eniInfo.IPv6Cidrs { - tmpENIInfo.IPv6Cidrs[cidr] = &CidrInfo{ + tmpENIInfo.IPv6Cidrs[cidr] = &v1.CidrInfo{ Cidr: eniInfo.IPv6Cidrs[cidr].Cidr, - IPAddresses: make(map[string]*AddressInfo, len(eniInfo.IPv6Cidrs[cidr].IPAddresses)), + IPAddresses: make(map[string]*v1.AddressInfo, len(eniInfo.IPv6Cidrs[cidr].IPAddresses)), IsPrefix: eniInfo.IPv6Cidrs[cidr].IsPrefix, } // Since IP Addresses might get removed, we need to make a deep copy here. @@ -1279,7 +1026,7 @@ func (ds *DataStore) GetFreePrefixes() int { } // getFreeIPv4AddrfromCidr returs a free IP/32 address from CIDR -func (ds *DataStore) getFreeIPv4AddrfromCidr(availableCidr *CidrInfo) (string, error) { +func (ds *DataStore) getFreeIPv4AddrfromCidr(availableCidr *v1.CidrInfo) (string, error) { if availableCidr == nil { ds.log.Errorf("Prefix datastore not initialized") return "", errors.New("Prefix datastore not initialized") @@ -1293,7 +1040,7 @@ func (ds *DataStore) getFreeIPv4AddrfromCidr(availableCidr *CidrInfo) (string, e return strPrivateIPv4, nil } -func (ds *DataStore) getFreeIPv6AddrFromCidr(IPv6Cidr *CidrInfo) (string, error) { +func (ds *DataStore) getFreeIPv6AddrFromCidr(IPv6Cidr *v1.CidrInfo) (string, error) { if IPv6Cidr == nil { ds.log.Errorf("Prefix datastore not initialized") return "", errors.New("Prefix datastore not initialized") @@ -1314,11 +1061,11 @@ func (ds *DataStore) getFreeIPv6AddrFromCidr(IPv6Cidr *CidrInfo) (string, error) return 10.1.1.1/32 */ -func (ds *DataStore) getUnusedIP(availableCidr *CidrInfo) (string, error) { +func (ds *DataStore) getUnusedIP(availableCidr *v1.CidrInfo) (string, error) { //Check if there is any IP out of cooldown var cachedIP string for _, addr := range availableCidr.IPAddresses { - if !addr.Assigned() && !addr.inCoolingPeriod(ds.ipCooldownPeriod) { + if !addr.Assigned() && !addr.InCoolingPeriod(ds.ipCooldownPeriod) { //if the IP is out of cooldown and not assigned then cache the first available IP //continue cleaning up the DB, this is to avoid stale entries and a new thread :) if cachedIP == "" { @@ -1369,7 +1116,7 @@ func GetPrefixDelegationDefaults() (int, int, int) { // FindFreeableCidrs finds and returns Cidrs that are not assigned to Pods but are attached // to ENIs on the node. -func (ds *DataStore) FindFreeableCidrs(eniID string) []CidrInfo { +func (ds *DataStore) FindFreeableCidrs(eniID string) []v1.CidrInfo { ds.lock.Lock() defer ds.lock.Unlock() @@ -1379,10 +1126,10 @@ func (ds *DataStore) FindFreeableCidrs(eniID string) []CidrInfo { return nil } - var freeable []CidrInfo + var freeable []v1.CidrInfo for _, assignedaddr := range eni.AvailableIPv4Cidrs { if assignedaddr.AssignedIPAddressesInCidr() == 0 { - tempFreeable := CidrInfo{ + tempFreeable := v1.CidrInfo{ Cidr: assignedaddr.Cidr, IPAddresses: nil, IsPrefix: assignedaddr.IsPrefix, @@ -1411,7 +1158,7 @@ func (ds *DataStore) CheckFreeableENIexists() bool { continue } - if eni.hasPods() { + if eni.HasPods() { ds.log.Debugf("ENI %s cannot be deleted because it has pods assigned", eni.ID) continue } @@ -1434,13 +1181,13 @@ func (ds *DataStore) CheckFreeableENIexists() bool { // NormalizeCheckpointDataByPodVethExistence will normalize checkpoint data by removing allocations that do not have a corresponding pod veth. // This can happen if pods are deleted while IPAMD is inactive. -func (ds *DataStore) normalizeCheckpointDataByPodVethExistence(checkpoint CheckpointData) (CheckpointData, error) { +func (ds *DataStore) normalizeCheckpointDataByPodVethExistence(checkpoint v1.CheckpointData) (v1.CheckpointData, error) { hostNSLinks, err := ds.netLink.LinkList() if err != nil { - return CheckpointData{}, err + return v1.CheckpointData{}, err } - var validatedAllocations []CheckpointEntry - var staleAllocations []CheckpointEntry + var validatedAllocations []v1.CheckpointEntry + var staleAllocations []v1.CheckpointEntry for _, allocation := range checkpoint.Allocations { if err := ds.validateAllocationByPodVethExistence(allocation, hostNSLinks); err != nil { ds.log.Warnf("stale IP allocation for ID(%v): IPv4(%v), IPv6(%v) due to %v", allocation.ContainerID, allocation.IPv4, allocation.IPv6, err) @@ -1457,7 +1204,7 @@ func (ds *DataStore) normalizeCheckpointDataByPodVethExistence(checkpoint Checkp return checkpoint, nil } -func (ds *DataStore) validateAllocationByPodVethExistence(allocation CheckpointEntry, hostNSLinks []netlink.Link) error { +func (ds *DataStore) validateAllocationByPodVethExistence(allocation v1.CheckpointEntry, hostNSLinks []netlink.Link) error { // for backwards compatibility, we skip the validation when metadata contains empty namespace/name. if allocation.Metadata.K8SPodNamespace == "" || allocation.Metadata.K8SPodName == "" { return nil @@ -1474,7 +1221,7 @@ func (ds *DataStore) validateAllocationByPodVethExistence(allocation CheckpointE } // For each stale allocation, cleanup leaked IP rules if they exist -func (ds *DataStore) PruneStaleAllocations(staleAllocations []CheckpointEntry) { +func (ds *DataStore) PruneStaleAllocations(staleAllocations []v1.CheckpointEntry) { ds.log.Info("Pruning potentially stale IP rules") for _, allocation := range staleAllocations { ds.DeleteToContainerRule(&allocation) @@ -1482,7 +1229,7 @@ func (ds *DataStore) PruneStaleAllocations(staleAllocations []CheckpointEntry) { } } -func (ds *DataStore) DeleteToContainerRule(entry *CheckpointEntry) { +func (ds *DataStore) DeleteToContainerRule(entry *v1.CheckpointEntry) { ds.log.Infof("Delete toContainer rule for v4: %s, v6: %s", entry.IPv4, entry.IPv6) // Remove toContainer rule, if it exists. Note that toContainer rule will always be in main routing table. toContainerRule := ds.netLink.NewRule() @@ -1508,7 +1255,7 @@ func (ds *DataStore) DeleteToContainerRule(entry *CheckpointEntry) { } } -func (ds *DataStore) DeleteFromContainerRule(entry *CheckpointEntry) { +func (ds *DataStore) DeleteFromContainerRule(entry *v1.CheckpointEntry) { ds.log.Infof("Delete fromContainer rule for v4: %s, v6: %s", entry.IPv4, entry.IPv6) // Remove fromContainer rule, if it exists. Note that fromContainer rule can be in any routing table, // so no table is set. diff --git a/pkg/ipamd/datastore/data_store_test.go b/pkg/ipamd/datastore/data_store_test.go index 3efd872dbd..36b2a14dfc 100644 --- a/pkg/ipamd/datastore/data_store_test.go +++ b/pkg/ipamd/datastore/data_store_test.go @@ -20,6 +20,7 @@ import ( "testing" "time" + v1 "github.com/aws/amazon-vpc-cni-k8s/pkg/ipamd/api/v1" mock_netlinkwrapper "github.com/aws/amazon-vpc-cni-k8s/pkg/netlinkwrapper/mocks" "github.com/aws/amazon-vpc-cni-k8s/pkg/networkutils" "github.com/aws/amazon-vpc-cni-k8s/utils/prometheusmetrics" @@ -94,8 +95,8 @@ func TestDeleteENI(t *testing.T) { err = ds.AddIPv4CidrToStore("eni-1", ipv4Addr, false) assert.NoError(t, err) ip, device, err := ds.AssignPodIPv4Address( - IPAMKey{"net1", "sandbox1", "eth0"}, - IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod"}) + v1.IPAMKey{NetworkName: "net1", ContainerID: "sandbox1", IfName: "eth0"}, + v1.IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod"}) assert.NoError(t, err) assert.Equal(t, "1.1.1.1", ip) assert.Equal(t, 1, device) @@ -144,8 +145,8 @@ func TestDeleteENIwithPDEnabled(t *testing.T) { err = ds.AddIPv4CidrToStore("eni-4", ipv4Addr, true) assert.NoError(t, err) ip, device, err := ds.AssignPodIPv4Address( - IPAMKey{"net1", "sandbox1", "eth0"}, - IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod"}) + v1.IPAMKey{NetworkName: "net1", ContainerID: "sandbox1", IfName: "eth0"}, + v1.IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod"}) assert.NoError(t, err) assert.Equal(t, "10.0.0.0", ip) assert.Equal(t, 4, device) @@ -329,8 +330,8 @@ func TestDelENIIPv4Address(t *testing.T) { assert.Equal(t, len(ds.eniPool["eni-1"].AvailableIPv4Cidrs), 1) // Assign a pod. - key := IPAMKey{"net0", "sandbox-1", "eth0"} - ip, device, err := ds.AssignPodIPv4Address(key, IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-1"}) + key := v1.IPAMKey{NetworkName: "net0", ContainerID: "sandbox-1", IfName: "eth0"} + ip, device, err := ds.AssignPodIPv4Address(key, v1.IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-1"}) assert.NoError(t, err) assert.Equal(t, "1.1.1.1", ip) assert.Equal(t, 1, device) @@ -387,8 +388,8 @@ func TestDelENIIPv4AddressWithPDEnabled(t *testing.T) { assert.Equal(t, len(ds.eniPool["eni-1"].AvailableIPv4Cidrs), 1) // Assign a pod. - key := IPAMKey{"net0", "sandbox-1", "eth0"} - ip, device, err := ds.AssignPodIPv4Address(key, IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-1"}) + key := v1.IPAMKey{NetworkName: "net0", ContainerID: "sandbox-1", IfName: "eth0"} + ip, device, err := ds.AssignPodIPv4Address(key, v1.IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-1"}) assert.NoError(t, err) assert.Equal(t, "10.0.0.0", ip) assert.Equal(t, 1, device) @@ -447,8 +448,8 @@ func TestTogglePD(t *testing.T) { assert.Equal(t, len(ds.eniPool["eni-1"].AvailableIPv4Cidrs), 1) // Assign a pod. - key := IPAMKey{"net0", "sandbox-1", "eth0"} - ip, device, err := ds.AssignPodIPv4Address(key, IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-1"}) + key := v1.IPAMKey{NetworkName: "net0", ContainerID: "sandbox-1", IfName: "eth0"} + ip, device, err := ds.AssignPodIPv4Address(key, v1.IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-1"}) assert.NoError(t, err) assert.Equal(t, "1.1.1.1", ip) assert.Equal(t, 1, device) @@ -464,8 +465,8 @@ func TestTogglePD(t *testing.T) { assert.Equal(t, len(ds.eniPool["eni-1"].AvailableIPv4Cidrs), 2) //Assign a pod - key = IPAMKey{"net0", "sandbox-2", "eth0"} - ip, device, err = ds.AssignPodIPv4Address(key, IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-2"}) + key = v1.IPAMKey{NetworkName: "net0", ContainerID: "sandbox-2", IfName: "eth0"} + ip, device, err = ds.AssignPodIPv4Address(key, v1.IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-2"}) assert.NoError(t, err) assert.Equal(t, "10.0.0.0", ip) assert.Equal(t, 1, device) @@ -517,8 +518,8 @@ func TestPodIPv4Address(t *testing.T) { ds := NewDataStore(Testlog, checkpoint, false) checkpointDataCmpOpts := cmp.Options{ - cmpopts.IgnoreFields(CheckpointEntry{}, "AllocationTimestamp"), - cmpopts.SortSlices(func(lhs CheckpointEntry, rhs CheckpointEntry) bool { + cmpopts.IgnoreFields(v1.CheckpointEntry{}, "AllocationTimestamp"), + cmpopts.SortSlices(func(lhs v1.CheckpointEntry, rhs v1.CheckpointEntry) bool { return lhs.ContainerID < rhs.ContainerID }), } @@ -533,8 +534,8 @@ func TestPodIPv4Address(t *testing.T) { err = ds.AddIPv4CidrToStore("eni-1", ipv4Addr1, false) assert.NoError(t, err) - key1 := IPAMKey{"net0", "sandbox-1", "eth0"} - ip, _, err := ds.AssignPodIPv4Address(key1, IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-1"}) + key1 := v1.IPAMKey{NetworkName: "net0", ContainerID: "sandbox-1", IfName: "eth0"} + ip, _, err := ds.AssignPodIPv4Address(key1, v1.IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-1"}) assert.NoError(t, err) assert.Equal(t, "1.1.1.1", ip) @@ -542,13 +543,13 @@ func TestPodIPv4Address(t *testing.T) { assert.Equal(t, 1, len(ds.eniPool["eni-1"].AvailableIPv4Cidrs)) assert.Equal(t, 1, ds.eniPool["eni-1"].AssignedIPv4Addresses()) - expectedCheckpointData := &CheckpointData{ - Version: CheckpointFormatVersion, - Allocations: []CheckpointEntry{ + expectedCheckpointData := &v1.CheckpointData{ + Version: v1.CheckpointFormatVersion, + Allocations: []v1.CheckpointEntry{ { - IPAMKey: IPAMKey{NetworkName: "net0", ContainerID: "sandbox-1", IfName: "eth0"}, + IPAMKey: v1.IPAMKey{NetworkName: "net0", ContainerID: "sandbox-1", IfName: "eth0"}, IPv4: "1.1.1.1", - Metadata: IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-1"}, + Metadata: v1.IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-1"}, }, }, } @@ -565,7 +566,7 @@ func TestPodIPv4Address(t *testing.T) { assert.NoError(t, err) // duplicate add - ip, _, err = ds.AssignPodIPv4Address(key1, IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-1"}) // same id + ip, _, err = ds.AssignPodIPv4Address(key1, v1.IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-1"}) // same id assert.NoError(t, err) assert.Equal(t, ip, "1.1.1.1") assert.Equal(t, ds.total, 2) @@ -577,17 +578,17 @@ func TestPodIPv4Address(t *testing.T) { // Checkpoint error checkpoint.Error = errors.New("fake checkpoint error") - key2 := IPAMKey{"net0", "sandbox-2", "eth0"} - _, _, err = ds.AssignPodIPv4Address(key2, IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-2"}) + key2 := v1.IPAMKey{NetworkName: "net0", ContainerID: "sandbox-2", IfName: "eth0"} + _, _, err = ds.AssignPodIPv4Address(key2, v1.IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-2"}) assert.Error(t, err) - expectedCheckpointData = &CheckpointData{ - Version: CheckpointFormatVersion, - Allocations: []CheckpointEntry{ + expectedCheckpointData = &v1.CheckpointData{ + Version: v1.CheckpointFormatVersion, + Allocations: []v1.CheckpointEntry{ { - IPAMKey: IPAMKey{NetworkName: "net0", ContainerID: "sandbox-1", IfName: "eth0"}, + IPAMKey: v1.IPAMKey{NetworkName: "net0", ContainerID: "sandbox-1", IfName: "eth0"}, IPv4: "1.1.1.1", - Metadata: IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-1"}, + Metadata: v1.IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-1"}, }, }, } @@ -597,7 +598,7 @@ func TestPodIPv4Address(t *testing.T) { ) checkpoint.Error = nil - ip, pod1Ns2Device, err := ds.AssignPodIPv4Address(key2, IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-2"}) + ip, pod1Ns2Device, err := ds.AssignPodIPv4Address(key2, v1.IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-2"}) assert.NoError(t, err) assert.Equal(t, ip, "1.1.2.2") assert.Equal(t, ds.total, 2) @@ -605,18 +606,18 @@ func TestPodIPv4Address(t *testing.T) { assert.Equal(t, len(ds.eniPool["eni-2"].AvailableIPv4Cidrs), 1) assert.Equal(t, ds.eniPool["eni-2"].AssignedIPv4Addresses(), 1) - expectedCheckpointData = &CheckpointData{ - Version: CheckpointFormatVersion, - Allocations: []CheckpointEntry{ + expectedCheckpointData = &v1.CheckpointData{ + Version: v1.CheckpointFormatVersion, + Allocations: []v1.CheckpointEntry{ { - IPAMKey: IPAMKey{NetworkName: "net0", ContainerID: "sandbox-1", IfName: "eth0"}, + IPAMKey: v1.IPAMKey{NetworkName: "net0", ContainerID: "sandbox-1", IfName: "eth0"}, IPv4: "1.1.1.1", - Metadata: IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-1"}, + Metadata: v1.IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-1"}, }, { - IPAMKey: IPAMKey{NetworkName: "net0", ContainerID: "sandbox-2", IfName: "eth0"}, + IPAMKey: v1.IPAMKey{NetworkName: "net0", ContainerID: "sandbox-2", IfName: "eth0"}, IPv4: "1.1.2.2", - Metadata: IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-2"}, + Metadata: v1.IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-2"}, }, }, } @@ -632,31 +633,30 @@ func TestPodIPv4Address(t *testing.T) { err = ds.AddIPv4CidrToStore("eni-1", ipv4Addr3, false) assert.NoError(t, err) - key3 := IPAMKey{"net0", "sandbox-3", "eth0"} - ip, _, err = ds.AssignPodIPv4Address(key3, IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-3"}) + key3 := v1.IPAMKey{NetworkName: "net0", ContainerID: "sandbox-3", IfName: "eth0"} + ip, _, err = ds.AssignPodIPv4Address(key3, v1.IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-3"}) assert.NoError(t, err) assert.Equal(t, ip, "1.1.1.2") assert.Equal(t, ds.total, 3) assert.Equal(t, ds.assigned, 3) assert.Equal(t, len(ds.eniPool["eni-1"].AvailableIPv4Cidrs), 2) assert.Equal(t, ds.eniPool["eni-1"].AssignedIPv4Addresses(), 2) - expectedCheckpointData = &CheckpointData{ - Version: CheckpointFormatVersion, - Allocations: []CheckpointEntry{ - { - IPAMKey: IPAMKey{NetworkName: "net0", ContainerID: "sandbox-1", IfName: "eth0"}, - IPv4: "1.1.1.1", - Metadata: IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-1"}, - }, + expectedCheckpointData = &v1.CheckpointData{ + Version: v1.CheckpointFormatVersion, + Allocations: []v1.CheckpointEntry{{ + IPAMKey: v1.IPAMKey{NetworkName: "net0", ContainerID: "sandbox-1", IfName: "eth0"}, + IPv4: "1.1.1.1", + Metadata: v1.IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-1"}, + }, { - IPAMKey: IPAMKey{NetworkName: "net0", ContainerID: "sandbox-2", IfName: "eth0"}, + IPAMKey: v1.IPAMKey{NetworkName: "net0", ContainerID: "sandbox-2", IfName: "eth0"}, IPv4: "1.1.2.2", - Metadata: IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-2"}, + Metadata: v1.IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-2"}, }, { - IPAMKey: IPAMKey{NetworkName: "net0", ContainerID: "sandbox-3", IfName: "eth0"}, + IPAMKey: v1.IPAMKey{NetworkName: "net0", ContainerID: "sandbox-3", IfName: "eth0"}, IPv4: "1.1.1.2", - Metadata: IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-3"}, + Metadata: v1.IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-3"}, }, }, } @@ -666,8 +666,8 @@ func TestPodIPv4Address(t *testing.T) { ) // no more IP addresses - key4 := IPAMKey{"net0", "sandbox-4", "eth0"} - _, _, err = ds.AssignPodIPv4Address(key4, IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-4"}) + key4 := v1.IPAMKey{NetworkName: "net0", ContainerID: "sandbox-4", IfName: "eth0"} + _, _, err = ds.AssignPodIPv4Address(key4, v1.IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-4"}) assert.Error(t, err) // Unassign unknown Pod _, _, _, err = ds.UnassignPodIPAddress(key4) @@ -680,18 +680,18 @@ func TestPodIPv4Address(t *testing.T) { assert.Equal(t, deviceNum, pod1Ns2Device) assert.Equal(t, len(ds.eniPool["eni-2"].AvailableIPv4Cidrs), 1) assert.Equal(t, ds.eniPool["eni-2"].AssignedIPv4Addresses(), 0) - expectedCheckpointData = &CheckpointData{ - Version: CheckpointFormatVersion, - Allocations: []CheckpointEntry{ + expectedCheckpointData = &v1.CheckpointData{ + Version: v1.CheckpointFormatVersion, + Allocations: []v1.CheckpointEntry{ { - IPAMKey: IPAMKey{NetworkName: "net0", ContainerID: "sandbox-1", IfName: "eth0"}, + IPAMKey: v1.IPAMKey{NetworkName: "net0", ContainerID: "sandbox-1", IfName: "eth0"}, IPv4: "1.1.1.1", - Metadata: IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-1"}, + Metadata: v1.IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-1"}, }, { - IPAMKey: IPAMKey{NetworkName: "net0", ContainerID: "sandbox-3", IfName: "eth0"}, + IPAMKey: v1.IPAMKey{NetworkName: "net0", ContainerID: "sandbox-3", IfName: "eth0"}, IPv4: "1.1.1.2", - Metadata: IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-3"}, + Metadata: v1.IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-3"}, }, }, } @@ -708,7 +708,7 @@ func TestPodIPv4Address(t *testing.T) { eni := ds.RemoveUnusedENIFromStore(noWarmIPTarget, noMinimumIPTarget, noWarmPrefixTarget) assert.True(t, eni == "") - ds.eniPool["eni-2"].createTime = time.Time{} + ds.eniPool["eni-2"].CreateTime = time.Time{} ds.eniPool["eni-2"].AvailableIPv4Cidrs[ipv4Addr2.String()].IPAddresses["1.1.2.2"].UnassignedTime = time.Time{} eni = ds.RemoveUnusedENIFromStore(noWarmIPTarget, noMinimumIPTarget, noWarmPrefixTarget) assert.Equal(t, eni, "eni-2") @@ -722,8 +722,8 @@ func TestPodIPv4AddressWithPDEnabled(t *testing.T) { ds := NewDataStore(Testlog, checkpoint, true) checkpointDataCmpOpts := cmp.Options{ - cmpopts.IgnoreFields(CheckpointEntry{}, "AllocationTimestamp"), - cmpopts.SortSlices(func(lhs CheckpointEntry, rhs CheckpointEntry) bool { + cmpopts.IgnoreFields(v1.CheckpointEntry{}, "AllocationTimestamp"), + cmpopts.SortSlices(func(lhs v1.CheckpointEntry, rhs v1.CheckpointEntry) bool { return lhs.ContainerID < rhs.ContainerID }), } @@ -738,8 +738,8 @@ func TestPodIPv4AddressWithPDEnabled(t *testing.T) { err = ds.AddIPv4CidrToStore("eni-1", ipv4Addr1, true) assert.NoError(t, err) - key1 := IPAMKey{"net0", "sandbox-1", "eth0"} - ip, _, err := ds.AssignPodIPv4Address(key1, IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-1"}) + key1 := v1.IPAMKey{NetworkName: "net0", ContainerID: "sandbox-1", IfName: "eth0"} + ip, _, err := ds.AssignPodIPv4Address(key1, v1.IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-1"}) assert.NoError(t, err) assert.Equal(t, "10.0.0.0", ip) @@ -747,13 +747,13 @@ func TestPodIPv4AddressWithPDEnabled(t *testing.T) { assert.Equal(t, 1, len(ds.eniPool["eni-1"].AvailableIPv4Cidrs)) assert.Equal(t, 1, ds.eniPool["eni-1"].AssignedIPv4Addresses()) - expectedCheckpointData := &CheckpointData{ - Version: CheckpointFormatVersion, - Allocations: []CheckpointEntry{ + expectedCheckpointData := &v1.CheckpointData{ + Version: v1.CheckpointFormatVersion, + Allocations: []v1.CheckpointEntry{ { - IPAMKey: IPAMKey{NetworkName: "net0", ContainerID: "sandbox-1", IfName: "eth0"}, + IPAMKey: v1.IPAMKey{NetworkName: "net0", ContainerID: "sandbox-1", IfName: "eth0"}, IPv4: "10.0.0.0", - Metadata: IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-1"}, + Metadata: v1.IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-1"}, }, }, } @@ -766,7 +766,7 @@ func TestPodIPv4AddressWithPDEnabled(t *testing.T) { assert.Equal(t, len(podsInfos), 1) // duplicate add - ip, _, err = ds.AssignPodIPv4Address(key1, IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-1"}) // same id + ip, _, err = ds.AssignPodIPv4Address(key1, v1.IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-1"}) // same id assert.NoError(t, err) assert.Equal(t, ip, "10.0.0.0") assert.Equal(t, ds.total, 16) @@ -776,17 +776,17 @@ func TestPodIPv4AddressWithPDEnabled(t *testing.T) { // Checkpoint error checkpoint.Error = errors.New("fake checkpoint error") - key2 := IPAMKey{"net0", "sandbox-2", "eth0"} - _, _, err = ds.AssignPodIPv4Address(key2, IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-2"}) + key2 := v1.IPAMKey{NetworkName: "net0", ContainerID: "sandbox-2", IfName: "eth0"} + _, _, err = ds.AssignPodIPv4Address(key2, v1.IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-2"}) assert.Error(t, err) - expectedCheckpointData = &CheckpointData{ - Version: CheckpointFormatVersion, - Allocations: []CheckpointEntry{ + expectedCheckpointData = &v1.CheckpointData{ + Version: v1.CheckpointFormatVersion, + Allocations: []v1.CheckpointEntry{ { - IPAMKey: IPAMKey{NetworkName: "net0", ContainerID: "sandbox-1", IfName: "eth0"}, + IPAMKey: v1.IPAMKey{NetworkName: "net0", ContainerID: "sandbox-1", IfName: "eth0"}, IPv4: "10.0.0.0", - Metadata: IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-1"}, + Metadata: v1.IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-1"}, }, }, } @@ -796,7 +796,7 @@ func TestPodIPv4AddressWithPDEnabled(t *testing.T) { ) checkpoint.Error = nil - ip, pod1Ns2Device, err := ds.AssignPodIPv4Address(key2, IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-2"}) + ip, pod1Ns2Device, err := ds.AssignPodIPv4Address(key2, v1.IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-2"}) assert.NoError(t, err) assert.Equal(t, ip, "10.0.0.1") assert.Equal(t, ds.total, 16) @@ -804,18 +804,18 @@ func TestPodIPv4AddressWithPDEnabled(t *testing.T) { assert.Equal(t, len(ds.eniPool["eni-1"].AvailableIPv4Cidrs), 1) assert.Equal(t, ds.eniPool["eni-1"].AssignedIPv4Addresses(), 2) - expectedCheckpointData = &CheckpointData{ - Version: CheckpointFormatVersion, - Allocations: []CheckpointEntry{ + expectedCheckpointData = &v1.CheckpointData{ + Version: v1.CheckpointFormatVersion, + Allocations: []v1.CheckpointEntry{ { - IPAMKey: IPAMKey{NetworkName: "net0", ContainerID: "sandbox-1", IfName: "eth0"}, + IPAMKey: v1.IPAMKey{NetworkName: "net0", ContainerID: "sandbox-1", IfName: "eth0"}, IPv4: "10.0.0.0", - Metadata: IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-1"}, + Metadata: v1.IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-1"}, }, { - IPAMKey: IPAMKey{NetworkName: "net0", ContainerID: "sandbox-2", IfName: "eth0"}, + IPAMKey: v1.IPAMKey{NetworkName: "net0", ContainerID: "sandbox-2", IfName: "eth0"}, IPv4: "10.0.0.1", - Metadata: IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-2"}, + Metadata: v1.IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-2"}, }, }, } @@ -827,31 +827,31 @@ func TestPodIPv4AddressWithPDEnabled(t *testing.T) { podsInfos = ds.AllocatedIPs() assert.Equal(t, len(podsInfos), 2) - key3 := IPAMKey{"net0", "sandbox-3", "eth0"} - ip, _, err = ds.AssignPodIPv4Address(key3, IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-3"}) + key3 := v1.IPAMKey{NetworkName: "net0", ContainerID: "sandbox-3", IfName: "eth0"} + ip, _, err = ds.AssignPodIPv4Address(key3, v1.IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-3"}) assert.NoError(t, err) assert.Equal(t, ip, "10.0.0.2") assert.Equal(t, ds.total, 16) assert.Equal(t, ds.assigned, 3) assert.Equal(t, len(ds.eniPool["eni-1"].AvailableIPv4Cidrs), 1) assert.Equal(t, ds.eniPool["eni-1"].AssignedIPv4Addresses(), 3) - expectedCheckpointData = &CheckpointData{ - Version: CheckpointFormatVersion, - Allocations: []CheckpointEntry{ + expectedCheckpointData = &v1.CheckpointData{ + Version: v1.CheckpointFormatVersion, + Allocations: []v1.CheckpointEntry{ { - IPAMKey: IPAMKey{NetworkName: "net0", ContainerID: "sandbox-1", IfName: "eth0"}, + IPAMKey: v1.IPAMKey{NetworkName: "net0", ContainerID: "sandbox-1", IfName: "eth0"}, IPv4: "10.0.0.0", - Metadata: IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-1"}, + Metadata: v1.IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-1"}, }, { - IPAMKey: IPAMKey{NetworkName: "net0", ContainerID: "sandbox-2", IfName: "eth0"}, + IPAMKey: v1.IPAMKey{NetworkName: "net0", ContainerID: "sandbox-2", IfName: "eth0"}, IPv4: "10.0.0.1", - Metadata: IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-2"}, + Metadata: v1.IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-2"}, }, { - IPAMKey: IPAMKey{NetworkName: "net0", ContainerID: "sandbox-3", IfName: "eth0"}, + IPAMKey: v1.IPAMKey{NetworkName: "net0", ContainerID: "sandbox-3", IfName: "eth0"}, IPv4: "10.0.0.2", - Metadata: IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-3"}, + Metadata: v1.IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-3"}, }, }, } @@ -867,18 +867,18 @@ func TestPodIPv4AddressWithPDEnabled(t *testing.T) { assert.Equal(t, deviceNum, pod1Ns2Device) assert.Equal(t, len(ds.eniPool["eni-1"].AvailableIPv4Cidrs), 1) assert.Equal(t, ds.eniPool["eni-1"].AssignedIPv4Addresses(), 2) - expectedCheckpointData = &CheckpointData{ - Version: CheckpointFormatVersion, - Allocations: []CheckpointEntry{ + expectedCheckpointData = &v1.CheckpointData{ + Version: v1.CheckpointFormatVersion, + Allocations: []v1.CheckpointEntry{ { - IPAMKey: IPAMKey{NetworkName: "net0", ContainerID: "sandbox-1", IfName: "eth0"}, + IPAMKey: v1.IPAMKey{NetworkName: "net0", ContainerID: "sandbox-1", IfName: "eth0"}, IPv4: "10.0.0.0", - Metadata: IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-1"}, + Metadata: v1.IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-1"}, }, { - IPAMKey: IPAMKey{NetworkName: "net0", ContainerID: "sandbox-3", IfName: "eth0"}, + IPAMKey: v1.IPAMKey{NetworkName: "net0", ContainerID: "sandbox-3", IfName: "eth0"}, IPv4: "10.0.0.2", - Metadata: IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-3"}, + Metadata: v1.IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-3"}, }, }, } @@ -900,18 +900,18 @@ func TestGetIPStatsV4(t *testing.T) { ipv4Addr := net.IPNet{IP: net.ParseIP("1.1.1.1"), Mask: net.IPv4Mask(255, 255, 255, 255)} _ = ds.AddIPv4CidrToStore("eni-1", ipv4Addr, false) - key1 := IPAMKey{"net0", "sandbox-1", "eth0"} - _, _, err := ds.AssignPodIPv4Address(key1, IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-1"}) + key1 := v1.IPAMKey{NetworkName: "net0", ContainerID: "sandbox-1", IfName: "eth0"} + _, _, err := ds.AssignPodIPv4Address(key1, v1.IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-1"}) assert.NoError(t, err) ipv4Addr = net.IPNet{IP: net.ParseIP("1.1.1.2"), Mask: net.IPv4Mask(255, 255, 255, 255)} _ = ds.AddIPv4CidrToStore("eni-1", ipv4Addr, false) - key2 := IPAMKey{"net0", "sandbox-2", "eth0"} - _, _, err = ds.AssignPodIPv4Address(key2, IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-2"}) + key2 := v1.IPAMKey{NetworkName: "net0", ContainerID: "sandbox-2", IfName: "eth0"} + _, _, err = ds.AssignPodIPv4Address(key2, v1.IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-2"}) assert.NoError(t, err) assert.Equal(t, - DataStoreStats{ + v1.DataStoreStats{ TotalIPs: 2, AssignedIPs: 2, CooldownIPs: 0, @@ -923,7 +923,7 @@ func TestGetIPStatsV4(t *testing.T) { assert.NoError(t, err) assert.Equal(t, - DataStoreStats{ + v1.DataStoreStats{ TotalIPs: 2, AssignedIPs: 1, CooldownIPs: 1, @@ -936,7 +936,7 @@ func TestGetIPStatsV4(t *testing.T) { time.Sleep(ds.ipCooldownPeriod) assert.Equal(t, - DataStoreStats{ + v1.DataStoreStats{ TotalIPs: 2, AssignedIPs: 1, CooldownIPs: 0, @@ -954,16 +954,16 @@ func TestGetIPStatsV4WithPD(t *testing.T) { ipv4Addr := net.IPNet{IP: net.ParseIP("10.0.0.0"), Mask: net.IPv4Mask(255, 255, 255, 240)} _ = ds.AddIPv4CidrToStore("eni-1", ipv4Addr, true) - key1 := IPAMKey{"net0", "sandbox-1", "eth0"} - _, _, err := ds.AssignPodIPv4Address(key1, IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-1"}) + key1 := v1.IPAMKey{NetworkName: "net0", ContainerID: "sandbox-1", IfName: "eth0"} + _, _, err := ds.AssignPodIPv4Address(key1, v1.IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-1"}) assert.NoError(t, err) - key2 := IPAMKey{"net0", "sandbox-2", "eth0"} - _, _, err = ds.AssignPodIPv4Address(key2, IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-2"}) + key2 := v1.IPAMKey{NetworkName: "net0", ContainerID: "sandbox-2", IfName: "eth0"} + _, _, err = ds.AssignPodIPv4Address(key2, v1.IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-2"}) assert.NoError(t, err) assert.Equal(t, - DataStoreStats{ + v1.DataStoreStats{ TotalIPs: 16, TotalPrefixes: 1, AssignedIPs: 2, @@ -976,7 +976,7 @@ func TestGetIPStatsV4WithPD(t *testing.T) { assert.NoError(t, err) assert.Equal(t, - DataStoreStats{ + v1.DataStoreStats{ TotalIPs: 16, TotalPrefixes: 1, AssignedIPs: 1, @@ -990,7 +990,7 @@ func TestGetIPStatsV4WithPD(t *testing.T) { time.Sleep(ds.ipCooldownPeriod) assert.Equal(t, - DataStoreStats{ + v1.DataStoreStats{ TotalIPs: 16, TotalPrefixes: 1, AssignedIPs: 1, @@ -1005,12 +1005,12 @@ func TestGetIPStatsV6(t *testing.T) { _ = v6ds.AddENI("eni-1", 1, true, false, false) ipv6Addr := net.IPNet{IP: net.IP{0x21, 0xdb, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, Mask: net.CIDRMask(80, 128)} _ = v6ds.AddIPv6CidrToStore("eni-1", ipv6Addr, true) - key3 := IPAMKey{"netv6", "sandbox-3", "eth0"} - _, _, err := v6ds.AssignPodIPv6Address(key3, IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-3"}) + key3 := v1.IPAMKey{NetworkName: "netv6", ContainerID: "sandbox-3", IfName: "eth0"} + _, _, err := v6ds.AssignPodIPv6Address(key3, v1.IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-3"}) assert.NoError(t, err) assert.Equal(t, - DataStoreStats{ + v1.DataStoreStats{ TotalIPs: 281474976710656, TotalPrefixes: 1, AssignedIPs: 1, @@ -1030,15 +1030,15 @@ func TestWarmENIInteractions(t *testing.T) { // Add an IP address to ENI 1 and assign it to a pod ipv4Addr := net.IPNet{IP: net.ParseIP("1.1.1.1"), Mask: net.IPv4Mask(255, 255, 255, 255)} _ = ds.AddIPv4CidrToStore("eni-1", ipv4Addr, false) - key1 := IPAMKey{"net0", "sandbox-1", "eth0"} - _, _, err := ds.AssignPodIPv4Address(key1, IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-1"}) + key1 := v1.IPAMKey{NetworkName: "net0", ContainerID: "sandbox-1", IfName: "eth0"} + _, _, err := ds.AssignPodIPv4Address(key1, v1.IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-1"}) assert.NoError(t, err) // Add another IP address to ENI 1 and assign a pod ipv4Addr = net.IPNet{IP: net.ParseIP("1.1.1.2"), Mask: net.IPv4Mask(255, 255, 255, 255)} _ = ds.AddIPv4CidrToStore("eni-1", ipv4Addr, false) - key2 := IPAMKey{"net0", "sandbox-2", "eth0"} - _, _, err = ds.AssignPodIPv4Address(key2, IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-2"}) + key2 := v1.IPAMKey{NetworkName: "net0", ContainerID: "sandbox-2", IfName: "eth0"} + _, _, err = ds.AssignPodIPv4Address(key2, v1.IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-2"}) assert.NoError(t, err) // Add two IP addresses to ENI 2 and one IP address to ENI 3 @@ -1049,8 +1049,8 @@ func TestWarmENIInteractions(t *testing.T) { ipv4Addr = net.IPNet{IP: net.ParseIP("1.1.3.1"), Mask: net.IPv4Mask(255, 255, 255, 255)} _ = ds.AddIPv4CidrToStore("eni-3", ipv4Addr, false) - ds.eniPool["eni-2"].createTime = time.Time{} - ds.eniPool["eni-3"].createTime = time.Time{} + ds.eniPool["eni-2"].CreateTime = time.Time{} + ds.eniPool["eni-3"].CreateTime = time.Time{} // We have 3 ENIs, 5 IPs and 2 pods on ENI 1. // ENI 1: 2 IPs allocated, 2 IPs in use @@ -1091,8 +1091,8 @@ func TestWarmENIInteractions(t *testing.T) { ds.AddIPv4CidrToStore("eni-4", ipv4Addr, false) ipv4Addr = net.IPNet{IP: net.ParseIP("1.1.5.1"), Mask: net.IPv4Mask(255, 255, 255, 255)} ds.AddIPv4CidrToStore("eni-5", ipv4Addr, false) - ds.eniPool["eni-4"].createTime = time.Time{} - ds.eniPool["eni-5"].createTime = time.Time{} + ds.eniPool["eni-4"].CreateTime = time.Time{} + ds.eniPool["eni-5"].CreateTime = time.Time{} // We have 3 ENIs, 4 IPs and 2 pods on ENI 1. // ENI 1: 2 IPs allocated, 2 IPs in use @@ -1107,7 +1107,7 @@ func TestWarmENIInteractions(t *testing.T) { // Add 1 more normal ENI to the datastore ds.AddENI("eni-6", 6, false, false, false) // trunk ENI - ds.eniPool["eni-6"].createTime = time.Time{} + ds.eniPool["eni-6"].CreateTime = time.Time{} // We have 4 ENIs, 4 IPs and 2 pods on ENI 1. // ENI 1: 2 IPs allocated, 2 IPs in use @@ -1147,13 +1147,13 @@ func TestDataStore_normalizeCheckpointDataByPodVethExistence(t *testing.T) { staleFromContainerRule *netlink.Rule } type args struct { - checkpoint CheckpointData + checkpoint v1.CheckpointData } tests := []struct { name string fields fields args args - want CheckpointData + want v1.CheckpointData wantErr error }{ { @@ -1182,29 +1182,29 @@ func TestDataStore_normalizeCheckpointDataByPodVethExistence(t *testing.T) { }, }, args: args{ - checkpoint: CheckpointData{ - Version: CheckpointFormatVersion, - Allocations: []CheckpointEntry{ + checkpoint: v1.CheckpointData{ + Version: v1.CheckpointFormatVersion, + Allocations: []v1.CheckpointEntry{ { - IPAMKey: IPAMKey{ + IPAMKey: v1.IPAMKey{ ContainerID: "5a1f9118a7125f87b4b0f2f601c0b55cfab8bcf28963bcf7c4ece3109a8b6b86", NetworkName: "aws-cni", IfName: "eth0", }, IPv4: "192.168.9.106", - Metadata: IPAMMetadata{ + Metadata: v1.IPAMMetadata{ K8SPodNamespace: "kube-system", K8SPodName: "coredns-57ff979f67-qqbdh", }, }, { - IPAMKey: IPAMKey{ + IPAMKey: v1.IPAMKey{ ContainerID: "b4729ce84caa9e585c2ba84fc9f9a058f4cf783292a84608f717035e09553422", NetworkName: "aws-cni", IfName: "eth0", }, IPv4: "192.168.30.161", - Metadata: IPAMMetadata{ + Metadata: v1.IPAMMetadata{ K8SPodNamespace: "kube-system", K8SPodName: "coredns-57ff979f67-8ns9b", }, @@ -1212,29 +1212,29 @@ func TestDataStore_normalizeCheckpointDataByPodVethExistence(t *testing.T) { }, }, }, - want: CheckpointData{ - Version: CheckpointFormatVersion, - Allocations: []CheckpointEntry{ + want: v1.CheckpointData{ + Version: v1.CheckpointFormatVersion, + Allocations: []v1.CheckpointEntry{ { - IPAMKey: IPAMKey{ + IPAMKey: v1.IPAMKey{ ContainerID: "5a1f9118a7125f87b4b0f2f601c0b55cfab8bcf28963bcf7c4ece3109a8b6b86", NetworkName: "aws-cni", IfName: "eth0", }, IPv4: "192.168.9.106", - Metadata: IPAMMetadata{ + Metadata: v1.IPAMMetadata{ K8SPodNamespace: "kube-system", K8SPodName: "coredns-57ff979f67-qqbdh", }, }, { - IPAMKey: IPAMKey{ + IPAMKey: v1.IPAMKey{ ContainerID: "b4729ce84caa9e585c2ba84fc9f9a058f4cf783292a84608f717035e09553422", NetworkName: "aws-cni", IfName: "eth0", }, IPv4: "192.168.30.161", - Metadata: IPAMMetadata{ + Metadata: v1.IPAMMetadata{ K8SPodNamespace: "kube-system", K8SPodName: "coredns-57ff979f67-8ns9b", }, @@ -1270,35 +1270,35 @@ func TestDataStore_normalizeCheckpointDataByPodVethExistence(t *testing.T) { staleFromContainerRule: fromContainerRule, }, args: args{ - checkpoint: CheckpointData{ - Version: CheckpointFormatVersion, - Allocations: []CheckpointEntry{ + checkpoint: v1.CheckpointData{ + Version: v1.CheckpointFormatVersion, + Allocations: []v1.CheckpointEntry{ { - IPAMKey: IPAMKey{ + IPAMKey: v1.IPAMKey{ ContainerID: "5a1f9118a7125f87b4b0f2f601c0b55cfab8bcf28963bcf7c4ece3109a8b6b86", NetworkName: "aws-cni", IfName: "eth0", }, IPv4: "192.168.9.106", - Metadata: IPAMMetadata{ + Metadata: v1.IPAMMetadata{ K8SPodNamespace: "kube-system", K8SPodName: "coredns-57ff979f67-qqbdh", }, }, { - IPAMKey: IPAMKey{ + IPAMKey: v1.IPAMKey{ ContainerID: "b4729ce84caa9e585c2ba84fc9f9a058f4cf783292a84608f717035e09553422", NetworkName: "aws-cni", IfName: "eth0", }, IPv4: "192.168.1.1", - Metadata: IPAMMetadata{ + Metadata: v1.IPAMMetadata{ K8SPodNamespace: "kube-system", K8SPodName: "coredns-57ff979f67-8ns9b", }, }, { - IPAMKey: IPAMKey{ + IPAMKey: v1.IPAMKey{ ContainerID: "d0437ce84caa9e585c2ba84fc9f9a058f4cf783292a84608f717035e0952341", NetworkName: "aws-cni", IfName: "eth0", @@ -1308,23 +1308,23 @@ func TestDataStore_normalizeCheckpointDataByPodVethExistence(t *testing.T) { }, }, }, - want: CheckpointData{ - Version: CheckpointFormatVersion, - Allocations: []CheckpointEntry{ + want: v1.CheckpointData{ + Version: v1.CheckpointFormatVersion, + Allocations: []v1.CheckpointEntry{ { - IPAMKey: IPAMKey{ + IPAMKey: v1.IPAMKey{ ContainerID: "5a1f9118a7125f87b4b0f2f601c0b55cfab8bcf28963bcf7c4ece3109a8b6b86", NetworkName: "aws-cni", IfName: "eth0", }, IPv4: "192.168.9.106", - Metadata: IPAMMetadata{ + Metadata: v1.IPAMMetadata{ K8SPodNamespace: "kube-system", K8SPodName: "coredns-57ff979f67-qqbdh", }, }, { - IPAMKey: IPAMKey{ + IPAMKey: v1.IPAMKey{ ContainerID: "d0437ce84caa9e585c2ba84fc9f9a058f4cf783292a84608f717035e0952341", NetworkName: "aws-cni", IfName: "eth0", @@ -1344,29 +1344,29 @@ func TestDataStore_normalizeCheckpointDataByPodVethExistence(t *testing.T) { }, }, args: args{ - checkpoint: CheckpointData{ - Version: CheckpointFormatVersion, - Allocations: []CheckpointEntry{ + checkpoint: v1.CheckpointData{ + Version: v1.CheckpointFormatVersion, + Allocations: []v1.CheckpointEntry{ { - IPAMKey: IPAMKey{ + IPAMKey: v1.IPAMKey{ ContainerID: "5a1f9118a7125f87b4b0f2f601c0b55cfab8bcf28963bcf7c4ece3109a8b6b86", NetworkName: "aws-cni", IfName: "eth0", }, IPv4: "192.168.9.106", - Metadata: IPAMMetadata{ + Metadata: v1.IPAMMetadata{ K8SPodNamespace: "kube-system", K8SPodName: "coredns-57ff979f67-qqbdh", }, }, { - IPAMKey: IPAMKey{ + IPAMKey: v1.IPAMKey{ ContainerID: "b4729ce84caa9e585c2ba84fc9f9a058f4cf783292a84608f717035e09553422", NetworkName: "aws-cni", IfName: "eth0", }, IPv4: "192.168.30.161", - Metadata: IPAMMetadata{ + Metadata: v1.IPAMMetadata{ K8SPodNamespace: "kube-system", K8SPodName: "coredns-57ff979f67-8ns9b", }, @@ -1407,7 +1407,7 @@ func TestDataStore_normalizeCheckpointDataByPodVethExistence(t *testing.T) { func TestDataStore_validateAllocationByPodVethExistence(t *testing.T) { type args struct { - allocation CheckpointEntry + allocation v1.CheckpointEntry hostNSLinks []netlink.Link } tests := []struct { @@ -1418,14 +1418,14 @@ func TestDataStore_validateAllocationByPodVethExistence(t *testing.T) { { name: "one veth pair found with matching suffix and eni prefix", args: args{ - allocation: CheckpointEntry{ - IPAMKey: IPAMKey{ + allocation: v1.CheckpointEntry{ + IPAMKey: v1.IPAMKey{ ContainerID: "5a1f9118a7125f87b4b0f2f601c0b55cfab8bcf28963bcf7c4ece3109a8b6b86", NetworkName: "aws-cni", IfName: "eth0", }, IPv4: "192.168.9.106", - Metadata: IPAMMetadata{ + Metadata: v1.IPAMMetadata{ K8SPodNamespace: "kube-system", K8SPodName: "coredns-57ff979f67-qqbdh", }, @@ -1448,14 +1448,14 @@ func TestDataStore_validateAllocationByPodVethExistence(t *testing.T) { { name: "one veth pair found with matching suffix and custom prefix", args: args{ - allocation: CheckpointEntry{ - IPAMKey: IPAMKey{ + allocation: v1.CheckpointEntry{ + IPAMKey: v1.IPAMKey{ ContainerID: "5a1f9118a7125f87b4b0f2f601c0b55cfab8bcf28963bcf7c4ece3109a8b6b86", NetworkName: "aws-cni", IfName: "eth0", }, IPv4: "192.168.9.106", - Metadata: IPAMMetadata{ + Metadata: v1.IPAMMetadata{ K8SPodNamespace: "kube-system", K8SPodName: "coredns-57ff979f67-qqbdh", }, @@ -1478,14 +1478,14 @@ func TestDataStore_validateAllocationByPodVethExistence(t *testing.T) { { name: "no veth pair found with matching suffix", args: args{ - allocation: CheckpointEntry{ - IPAMKey: IPAMKey{ + allocation: v1.CheckpointEntry{ + IPAMKey: v1.IPAMKey{ ContainerID: "5a1f9118a7125f87b4b0f2f601c0b55cfab8bcf28963bcf7c4ece3109a8b6b86", NetworkName: "aws-cni", IfName: "eth0", }, IPv4: "192.168.9.106", - Metadata: IPAMMetadata{ + Metadata: v1.IPAMMetadata{ K8SPodNamespace: "kube-system", K8SPodName: "coredns-57ff979f67-qqbdh", }, @@ -1508,14 +1508,14 @@ func TestDataStore_validateAllocationByPodVethExistence(t *testing.T) { { name: "allocation without metadata should pass validation", args: args{ - allocation: CheckpointEntry{ - IPAMKey: IPAMKey{ + allocation: v1.CheckpointEntry{ + IPAMKey: v1.IPAMKey{ ContainerID: "5a1f9118a7125f87b4b0f2f601c0b55cfab8bcf28963bcf7c4ece3109a8b6b86", NetworkName: "aws-cni", IfName: "eth0", }, IPv4: "192.168.9.106", - Metadata: IPAMMetadata{}, + Metadata: v1.IPAMMetadata{}, }, hostNSLinks: []netlink.Link{ &netlink.Device{ @@ -1568,8 +1568,8 @@ func TestForceRemovalMetrics(t *testing.T) { assert.NoError(t, err) // Assign IP to a pod - key := IPAMKey{"net0", "sandbox-1", "eth0"} - ip, device, err := ds.AssignPodIPv4Address(key, IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-1"}) + key := v1.IPAMKey{NetworkName: "net0", ContainerID: "sandbox-1", IfName: "eth0"} + ip, device, err := ds.AssignPodIPv4Address(key, v1.IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-1"}) assert.NoError(t, err) assert.Equal(t, "1.1.1.1", ip) assert.Equal(t, 1, device) @@ -1594,8 +1594,8 @@ func TestForceRemovalMetrics(t *testing.T) { err = ds.AddIPv4CidrToStore("eni-1", ipv4Addr2, false) assert.NoError(t, err) - key2 := IPAMKey{"net0", "sandbox-2", "eth0"} - ip, device, err = ds.AssignPodIPv4Address(key2, IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-2"}) + key2 := v1.IPAMKey{NetworkName: "net0", ContainerID: "sandbox-2", IfName: "eth0"} + ip, device, err = ds.AssignPodIPv4Address(key2, v1.IPAMMetadata{K8SPodNamespace: "default", K8SPodName: "sample-pod-2"}) assert.NoError(t, err) assert.Equal(t, "1.1.1.2", ip) assert.Equal(t, 1, device) diff --git a/pkg/ipamd/ipamd.go b/pkg/ipamd/ipamd.go index d569161d1d..20207d47e2 100644 --- a/pkg/ipamd/ipamd.go +++ b/pkg/ipamd/ipamd.go @@ -43,6 +43,7 @@ import ( "github.com/aws/amazon-vpc-cni-k8s/pkg/awsutils" "github.com/aws/amazon-vpc-cni-k8s/pkg/eniconfig" + v1 "github.com/aws/amazon-vpc-cni-k8s/pkg/ipamd/api/v1" "github.com/aws/amazon-vpc-cni-k8s/pkg/ipamd/datastore" "github.com/aws/amazon-vpc-cni-k8s/pkg/networkutils" "github.com/aws/amazon-vpc-cni-k8s/pkg/utils/cniutils" @@ -789,7 +790,7 @@ func (c *IPAMContext) tryUnassignCidrsFromAll() { } // Delete IPs from datastore - var deletedCidrs []datastore.CidrInfo + var deletedCidrs []v1.CidrInfo for _, toDelete := range cidrs { // Do not force the delete, since a freeable Cidr might have been assigned to a pod // before we get around to deleting it. @@ -1255,7 +1256,7 @@ func getWarmPrefixTarget() int { } // logPoolStats logs usage information for allocated addresses/prefixes. -func (c *IPAMContext) logPoolStats(dataStoreStats *datastore.DataStoreStats) { +func (c *IPAMContext) logPoolStats(dataStoreStats *v1.DataStoreStats) { prefix := "IP pool stats" if c.enablePrefixDelegation { prefix = "Prefix pool stats" @@ -1905,7 +1906,7 @@ func (c *IPAMContext) filterUnmanagedENIs(enis []awsutils.ENIMetadata) []awsutil // datastoreTargetState determines the number of IPs `short` or `over` our WARM_IP_TARGET, accounting for the MINIMUM_IP_TARGET. // With prefix delegation, this function determines the number of Prefixes `short` or `over` -func (c *IPAMContext) datastoreTargetState(stats *datastore.DataStoreStats) (short int, over int, enabled bool) { +func (c *IPAMContext) datastoreTargetState(stats *v1.DataStoreStats) (short int, over int, enabled bool) { if !c.warmIPTargetsDefined() { // there is no WARM_IP_TARGET defined and no MINIMUM_IP_TARGET, fallback to use all IP addresses on ENI return 0, 0, false @@ -2239,7 +2240,7 @@ func (c *IPAMContext) hasRoomForEni() bool { return c.dataStore.GetENIs() < (c.maxENI - c.unmanagedENI - trunkEni) } -func (c *IPAMContext) isDatastorePoolTooLow() (bool, *datastore.DataStoreStats) { +func (c *IPAMContext) isDatastorePoolTooLow() (bool, *v1.DataStoreStats) { stats := c.dataStore.GetIPStats(ipV4AddrFamily) // If max pods has been reached, pool is not too low if stats.TotalIPs >= c.maxPods { @@ -2268,7 +2269,7 @@ func (c *IPAMContext) isDatastorePoolTooLow() (bool, *datastore.DataStoreStats) return poolTooLow, stats } -func (c *IPAMContext) isDatastorePoolTooHigh(stats *datastore.DataStoreStats) bool { +func (c *IPAMContext) isDatastorePoolTooHigh(stats *v1.DataStoreStats) bool { // NOTE: IPs may be allocated in chunks (full ENIs of prefixes), so the "too-high" condition does not check max pods. The limit is enforced on the allocation side. _, over, warmTargetDefined := c.datastoreTargetState(stats) if warmTargetDefined { @@ -2293,7 +2294,7 @@ func (c *IPAMContext) warmPrefixTargetDefined() bool { } // DeallocCidrs frees IPs and Prefixes from EC2 -func (c *IPAMContext) DeallocCidrs(eniID string, deletableCidrs []datastore.CidrInfo) { +func (c *IPAMContext) DeallocCidrs(eniID string, deletableCidrs []v1.CidrInfo) { var deletableIPs []string var deletablePrefixes []string diff --git a/pkg/ipamd/ipamd_test.go b/pkg/ipamd/ipamd_test.go index e14bce5000..7f75e8e316 100644 --- a/pkg/ipamd/ipamd_test.go +++ b/pkg/ipamd/ipamd_test.go @@ -47,6 +47,7 @@ import ( "github.com/aws/amazon-vpc-cni-k8s/pkg/awsutils" mock_awsutils "github.com/aws/amazon-vpc-cni-k8s/pkg/awsutils/mocks" mock_eniconfig "github.com/aws/amazon-vpc-cni-k8s/pkg/eniconfig/mocks" + ipamdv1 "github.com/aws/amazon-vpc-cni-k8s/pkg/ipamd/api/v1" "github.com/aws/amazon-vpc-cni-k8s/pkg/ipamd/datastore" mock_networkutils "github.com/aws/amazon-vpc-cni-k8s/pkg/networkutils/mocks" "github.com/aws/amazon-vpc-cni-k8s/utils/prometheusmetrics" @@ -116,10 +117,10 @@ func TestNodeInit(t *testing.T) { defer m.ctrl.Finish() ctx := context.Background() - fakeCheckpoint := datastore.CheckpointData{ - Version: datastore.CheckpointFormatVersion, - Allocations: []datastore.CheckpointEntry{ - {IPAMKey: datastore.IPAMKey{NetworkName: "net0", ContainerID: "sandbox-id", IfName: "eth0"}, IPv4: ipaddr02}, + fakeCheckpoint := ipamdv1.CheckpointData{ + Version: ipamdv1.CheckpointFormatVersion, + Allocations: []ipamdv1.CheckpointEntry{ + {IPAMKey: ipamdv1.IPAMKey{NetworkName: "net0", ContainerID: "sandbox-id", IfName: "eth0"}, IPv4: ipaddr02}, }, } @@ -206,10 +207,10 @@ func TestNodeInitwithPDenabledIPv4Mode(t *testing.T) { defer m.ctrl.Finish() ctx := context.Background() - fakeCheckpoint := datastore.CheckpointData{ - Version: datastore.CheckpointFormatVersion, - Allocations: []datastore.CheckpointEntry{ - {IPAMKey: datastore.IPAMKey{NetworkName: "net0", ContainerID: "sandbox-id", IfName: "eth0"}, IPv4: ipaddrPD01}, + fakeCheckpoint := ipamdv1.CheckpointData{ + Version: ipamdv1.CheckpointFormatVersion, + Allocations: []ipamdv1.CheckpointEntry{ + {IPAMKey: ipamdv1.IPAMKey{NetworkName: "net0", ContainerID: "sandbox-id", IfName: "eth0"}, IPv4: ipaddrPD01}, }, } @@ -294,10 +295,10 @@ func TestNodeInitwithPDenabledIPv6Mode(t *testing.T) { defer m.ctrl.Finish() ctx := context.Background() - fakeCheckpoint := datastore.CheckpointData{ - Version: datastore.CheckpointFormatVersion, - Allocations: []datastore.CheckpointEntry{ - {IPAMKey: datastore.IPAMKey{NetworkName: "net0", ContainerID: "sandbox-id", IfName: "eth0"}, IPv6: ipaddrPD01}, + fakeCheckpoint := ipamdv1.CheckpointData{ + Version: ipamdv1.CheckpointFormatVersion, + Allocations: []ipamdv1.CheckpointEntry{ + {IPAMKey: ipamdv1.IPAMKey{NetworkName: "net0", ContainerID: "sandbox-id", IfName: "eth0"}, IPv6: ipaddrPD01}, }, } @@ -830,12 +831,12 @@ func TestDecreaseIPPool(t *testing.T) { mockContext.dataStore.AddENI(primaryENIid, primaryDevice, true, false, false) mockContext.dataStore.AddIPv4CidrToStore(primaryENIid, testAddr1, false) mockContext.dataStore.AddIPv4CidrToStore(primaryENIid, testAddr2, false) - mockContext.dataStore.AssignPodIPv4Address(datastore.IPAMKey{ContainerID: "container1"}, datastore.IPAMMetadata{K8SPodName: "pod1"}) + mockContext.dataStore.AssignPodIPv4Address(ipamdv1.IPAMKey{ContainerID: "container1"}, ipamdv1.IPAMMetadata{K8SPodName: "pod1"}) mockContext.dataStore.AddENI(secENIid, secDevice, true, false, false) mockContext.dataStore.AddIPv4CidrToStore(secENIid, testAddr11, false) mockContext.dataStore.AddIPv4CidrToStore(secENIid, testAddr12, false) - mockContext.dataStore.AssignPodIPv4Address(datastore.IPAMKey{ContainerID: "container2"}, datastore.IPAMMetadata{K8SPodName: "pod2"}) + mockContext.dataStore.AssignPodIPv4Address(ipamdv1.IPAMKey{ContainerID: "container2"}, ipamdv1.IPAMMetadata{K8SPodName: "pod2"}) m.awsutils.EXPECT().DeallocPrefixAddresses(gomock.Any(), gomock.Any()).Times(1) m.awsutils.EXPECT().DeallocIPAddresses(gomock.Any(), gomock.Any()).Times(1) @@ -1361,11 +1362,11 @@ func TestIPAMContext_nodePrefixPoolTooLow(t *testing.T) { } func testDatastore() *datastore.DataStore { - return datastore.NewDataStore(log, datastore.NewTestCheckpoint(datastore.CheckpointData{Version: datastore.CheckpointFormatVersion}), false) + return datastore.NewDataStore(log, datastore.NewTestCheckpoint(ipamdv1.CheckpointData{Version: ipamdv1.CheckpointFormatVersion}), false) } func testDatastorewithPrefix() *datastore.DataStore { - return datastore.NewDataStore(log, datastore.NewTestCheckpoint(datastore.CheckpointData{Version: datastore.CheckpointFormatVersion}), true) + return datastore.NewDataStore(log, datastore.NewTestCheckpoint(ipamdv1.CheckpointData{Version: ipamdv1.CheckpointFormatVersion}), true) } func datastoreWith3FreeIPs() *datastore.DataStore { @@ -1383,11 +1384,11 @@ func datastoreWith3FreeIPs() *datastore.DataStore { func datastoreWith1Pod1() *datastore.DataStore { datastoreWith1Pod1 := datastoreWith3FreeIPs() - _, _, _ = datastoreWith1Pod1.AssignPodIPv4Address(datastore.IPAMKey{ + _, _, _ = datastoreWith1Pod1.AssignPodIPv4Address(ipamdv1.IPAMKey{ NetworkName: "net0", ContainerID: "sandbox-1", IfName: "eth0", - }, datastore.IPAMMetadata{ + }, ipamdv1.IPAMMetadata{ K8SPodNamespace: "default", K8SPodName: "sample-pod", }) @@ -1398,12 +1399,12 @@ func datastoreWith3Pods() *datastore.DataStore { datastoreWith3Pods := datastoreWith3FreeIPs() for i := 0; i < 3; i++ { - key := datastore.IPAMKey{ + key := ipamdv1.IPAMKey{ NetworkName: "net0", ContainerID: fmt.Sprintf("sandbox-%d", i), IfName: "eth0", } - _, _, _ = datastoreWith3Pods.AssignPodIPv4Address(key, datastore.IPAMMetadata{ + _, _, _ = datastoreWith3Pods.AssignPodIPv4Address(key, ipamdv1.IPAMMetadata{ K8SPodNamespace: "default", K8SPodName: fmt.Sprintf("sample-pod-%d", i), }) @@ -1422,11 +1423,11 @@ func datastoreWithFreeIPsFromPrefix() *datastore.DataStore { func datastoreWith1Pod1FromPrefix() *datastore.DataStore { datastoreWith1Pod1 := datastoreWithFreeIPsFromPrefix() - _, _, _ = datastoreWith1Pod1.AssignPodIPv4Address(datastore.IPAMKey{ + _, _, _ = datastoreWith1Pod1.AssignPodIPv4Address(ipamdv1.IPAMKey{ NetworkName: "net0", ContainerID: "sandbox-1", IfName: "eth0", - }, datastore.IPAMMetadata{ + }, ipamdv1.IPAMMetadata{ K8SPodNamespace: "default", K8SPodName: "sample-pod", }) @@ -1437,13 +1438,13 @@ func datastoreWith3PodsFromPrefix() *datastore.DataStore { datastoreWith3Pods := datastoreWithFreeIPsFromPrefix() for i := 0; i < 3; i++ { - key := datastore.IPAMKey{ + key := ipamdv1.IPAMKey{ NetworkName: "net0", ContainerID: fmt.Sprintf("sandbox-%d", i), IfName: "eth0", } _, _, _ = datastoreWith3Pods.AssignPodIPv4Address(key, - datastore.IPAMMetadata{ + ipamdv1.IPAMMetadata{ K8SPodNamespace: "default", K8SPodName: fmt.Sprintf("sample-pod-%d", i), }) @@ -2029,7 +2030,7 @@ func TestIPAMContext_enableSecurityGroupsForPods(t *testing.T) { k8sClient: m.k8sClient, enableIPv4: true, enableIPv6: false, - dataStore: datastore.NewDataStore(log, datastore.NewTestCheckpoint(datastore.CheckpointData{Version: datastore.CheckpointFormatVersion}), false), + dataStore: datastore.NewDataStore(log, datastore.NewTestCheckpoint(ipamdv1.CheckpointData{Version: ipamdv1.CheckpointFormatVersion}), false), awsClient: m.awsutils, networkClient: m.network, primaryIP: make(map[string]string), diff --git a/pkg/ipamd/rpc_handler.go b/pkg/ipamd/rpc_handler.go index 236fa0f0a9..3eb70a1d86 100644 --- a/pkg/ipamd/rpc_handler.go +++ b/pkg/ipamd/rpc_handler.go @@ -32,6 +32,7 @@ import ( "google.golang.org/grpc/status" "google.golang.org/protobuf/types/known/emptypb" + v1 "github.com/aws/amazon-vpc-cni-k8s/pkg/ipamd/api/v1" "github.com/aws/amazon-vpc-cni-k8s/pkg/ipamd/datastore" "github.com/aws/amazon-vpc-cni-k8s/pkg/networkutils" "github.com/aws/amazon-vpc-cni-k8s/rpc" @@ -163,12 +164,12 @@ func (s *server) AddNetwork(ctx context.Context, in *rpc.AddNetworkRequest) (*rp log.Errorf("Unable to generate IPAMKey from %+v", in) return &failureResponse, nil } - ipamKey := datastore.IPAMKey{ + ipamKey := v1.IPAMKey{ ContainerID: in.ContainerID, IfName: in.IfName, NetworkName: in.NetworkName, } - ipamMetadata := datastore.IPAMMetadata{ + ipamMetadata := v1.IPAMMetadata{ K8SPodNamespace: in.K8S_POD_NAMESPACE, K8SPodName: in.K8S_POD_NAME, } @@ -254,7 +255,7 @@ func (s *server) DelNetwork(ctx context.Context, in *rpc.DelNetworkRequest) (*rp return nil, err } - ipamKey := datastore.IPAMKey{ + ipamKey := v1.IPAMKey{ ContainerID: in.ContainerID, IfName: in.IfName, NetworkName: in.NetworkName,