Skip to content

Commit 3f5ee43

Browse files
pkoshtavmwarepkoshtavmware
committed
AV-232614 Static Route fixes.
1 parent 1cde1fb commit 3f5ee43

File tree

7 files changed

+810
-155
lines changed

7 files changed

+810
-155
lines changed

‎.github/workflows/go.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ jobs:
1515
name: Golangci-lint
1616
strategy:
1717
matrix:
18-
platform: [ ubuntu-20.04 ]
18+
platform: [ ubuntu-24.04 ]
1919
runs-on: ${{ matrix.platform }}
2020
steps:
2121
- name: Set up Go 1.23.0

‎internal/lib/utils.go

+11
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,13 @@ import (
1818
"context"
1919
"encoding/json"
2020
"fmt"
21+
"log"
2122
"regexp"
2223
"strings"
2324
"sync"
2425

26+
"github.com/google/uuid"
27+
2528
routev1 "github.com/openshift/api/route/v1"
2629
corev1 "k8s.io/api/core/v1"
2730
networkingv1 "k8s.io/api/networking/v1"
@@ -594,3 +597,11 @@ func (s *LockSet) Unlock(lockName string) {
594597
func GetLockSet() *LockSet {
595598
return &lockSet
596599
}
600+
601+
func Uuid4() string {
602+
id, err := uuid.NewRandom()
603+
if err != nil {
604+
log.Fatal(err)
605+
}
606+
return id.String()
607+
}

‎internal/nodes/avi_model_nodes.go

+2-4
Original file line numberDiff line numberDiff line change
@@ -323,17 +323,15 @@ func (o *AviObjectGraph) GetOrderedNodes() []AviModelNode {
323323
}
324324

325325
type StaticRouteDetails struct {
326-
StartIndex int
327-
Count int
328-
routeID int
326+
Count int
327+
RouteIDPrefix string
329328
}
330329
type AviVrfNode struct {
331330
Name string
332331
StaticRoutes []*avimodels.StaticRoute
333332
CloudConfigCksum uint32
334333
NodeStaticRoutes map[string]StaticRouteDetails
335334
Nodes []string
336-
NodeIds map[int]struct{}
337335
}
338336

339337
func (v *AviVrfNode) GetCheckSum() uint32 {

‎internal/nodes/avi_vrf_translator.go

+96-84
Original file line numberDiff line numberDiff line change
@@ -16,19 +16,79 @@ package nodes
1616

1717
import (
1818
"errors"
19-
"math"
2019
"regexp"
20+
"sort"
2121
"strconv"
2222
"strings"
2323

2424
"github.com/vmware/load-balancer-and-ingress-services-for-kubernetes/internal/lib"
25-
25+
"github.com/vmware/load-balancer-and-ingress-services-for-kubernetes/internal/objects"
2626
"github.com/vmware/load-balancer-and-ingress-services-for-kubernetes/pkg/utils"
2727

2828
"github.com/vmware/alb-sdk/go/models"
2929
v1 "k8s.io/api/core/v1"
3030
)
3131

32+
func GetStaticRoutesForOtherNodes(aviVrfNode *AviVrfNode, routeId string) []*models.StaticRoute {
33+
var staticRouteCopy []*models.StaticRoute
34+
nodePrefix := lib.GetClusterName() + "-" + routeId
35+
for i := 0; i < len(aviVrfNode.StaticRoutes); i++ {
36+
if routeId == "" || !strings.HasPrefix(*aviVrfNode.StaticRoutes[i].RouteID, nodePrefix) {
37+
staticRouteCopy = append(staticRouteCopy, aviVrfNode.StaticRoutes[i])
38+
}
39+
}
40+
return staticRouteCopy
41+
}
42+
43+
func (o *AviObjectGraph) CheckAndDeduplicateRecords(key string) {
44+
aviVrfNodes := o.GetAviVRF()
45+
if len(aviVrfNodes) == 0 {
46+
return
47+
}
48+
// Each AKO should have single VRF node as it deals with single cluster only.
49+
aviVrfNode := aviVrfNodes[0]
50+
51+
podCidrNextHopMap := make(map[string]string)
52+
hasDuplicateRecords := false
53+
// Check if duplicate records for staticroutes exist
54+
for i := 0; i < len(aviVrfNode.StaticRoutes); i++ {
55+
_, ok := podCidrNextHopMap[*aviVrfNode.StaticRoutes[i].Prefix.IPAddr.Addr]
56+
if ok {
57+
utils.AviLog.Debug("VRFContext has duplicate records.")
58+
hasDuplicateRecords = true
59+
break
60+
} else {
61+
podCidrNextHopMap[*aviVrfNode.StaticRoutes[i].Prefix.IPAddr.Addr] = *aviVrfNode.StaticRoutes[i].NextHop.Addr
62+
}
63+
}
64+
if !hasDuplicateRecords {
65+
return
66+
}
67+
68+
utils.AviLog.Debug("Starting deduplication of records in VRFContext")
69+
70+
o.Lock.Lock()
71+
// Clean VRFCache
72+
aviVrfNode.Nodes = nil
73+
aviVrfNode.StaticRoutes = nil
74+
aviVrfNode.NodeStaticRoutes = nil
75+
76+
o.Lock.Unlock()
77+
78+
// send sorted list of nodes from here.
79+
allNodes := objects.SharedNodeLister().CopyAllObjects()
80+
var nodeNames []string
81+
for k := range allNodes {
82+
nodeNames = append(nodeNames, k)
83+
}
84+
sort.Strings(nodeNames)
85+
for _, nodeKey := range nodeNames {
86+
o.BuildVRFGraph(key, aviVrfNode.Name, nodeKey, false)
87+
}
88+
89+
utils.AviLog.Debug("Deduplication of records in VRFContext finished")
90+
}
91+
3292
// BuildVRFGraph : build vrf graph from k8s nodes
3393
func (o *AviObjectGraph) BuildVRFGraph(key, vrfName, nodeName string, deleteFlag bool) error {
3494
o.Lock.Lock()
@@ -45,10 +105,8 @@ func (o *AviObjectGraph) BuildVRFGraph(key, vrfName, nodeName string, deleteFlag
45105
// Each AKO should have single VRF node as it deals with single cluster only.
46106
aviVrfNode := aviVrfNodes[0]
47107

48-
routeid := 1
49108
if len(aviVrfNode.StaticRoutes) == 0 {
50109
aviVrfNode.NodeStaticRoutes = make(map[string]StaticRouteDetails)
51-
aviVrfNode.NodeIds = make(map[int]struct{})
52110
}
53111
var nodeRoutes []*models.StaticRoute
54112
// For new node addition (coming from ingestion layer), nodes static routes will be attahced at the end
@@ -60,19 +118,22 @@ func (o *AviObjectGraph) BuildVRFGraph(key, vrfName, nodeName string, deleteFlag
60118
node, err := utils.GetInformers().NodeInformer.Lister().Get(nodeName)
61119
if err != nil {
62120
utils.AviLog.Errorf("key: %s, Error in fetching node details: %s: %v", key, nodeName, err)
121+
aviVrfNode.StaticRoutes = GetStaticRoutesForOtherNodes(aviVrfNode, nodeStaticRouteDetails.RouteIDPrefix)
122+
processNodeStaticRouteAndNodeDeletion(nodeName, aviVrfNode)
63123
return err
64124
}
125+
var routeIdPrefix string
65126
if ok {
66-
routeid = nodeStaticRouteDetails.routeID
127+
routeIdPrefix = nodeStaticRouteDetails.RouteIDPrefix
67128
} else {
68-
//O(n) for each node. But it will re-use previous index. So used instead of always incrementing index.
69-
routeid = findFreeRouteId(aviVrfNode.NodeIds)
129+
routeIdPrefix = lib.Uuid4()
70130
}
71-
aviVrfNode.NodeIds[routeid] = struct{}{}
72-
nodeRoutes, err = o.addRouteForNode(node, vrfName, routeid, aviVrfNode.NodeIds)
131+
132+
nodeRoutes, err = o.addRouteForNode(node, vrfName, routeIdPrefix)
73133
if err != nil {
74134
utils.AviLog.Errorf("key: %s, Error Adding vrf for node %s: %v", key, nodeName, err)
75-
delete(aviVrfNode.NodeIds, routeid)
135+
aviVrfNode.StaticRoutes = GetStaticRoutesForOtherNodes(aviVrfNode, routeIdPrefix)
136+
processNodeStaticRouteAndNodeDeletion(nodeName, aviVrfNode)
76137
return err
77138
}
78139
if !ok {
@@ -81,62 +142,45 @@ func (o *AviObjectGraph) BuildVRFGraph(key, vrfName, nodeName string, deleteFlag
81142
// node is not present and no overlapping of cidr, append at last
82143
aviVrfNode.StaticRoutes = append(aviVrfNode.StaticRoutes, nodeRoutes...)
83144
nodeStaticRoute := StaticRouteDetails{}
84-
// start index shows at what index of StaticRoutes, nodes routes start (index based zero)
85-
nodeStaticRoute.StartIndex = routeid - 1
86145
nodeStaticRoute.Count = len(nodeRoutes)
87-
nodeStaticRoute.routeID = routeid
146+
nodeStaticRoute.RouteIDPrefix = routeIdPrefix
88147
aviVrfNode.NodeStaticRoutes[nodeName] = nodeStaticRoute
89148
aviVrfNode.Nodes = append(aviVrfNode.Nodes, nodeName)
90149
} else {
91150
if len(nodeRoutes) == 0 {
92-
delete(aviVrfNode.NodeIds, routeid)
151+
//delete all the routes and details of this node
152+
aviVrfNode.StaticRoutes = GetStaticRoutesForOtherNodes(aviVrfNode, routeIdPrefix)
153+
processNodeStaticRouteAndNodeDeletion(nodeName, aviVrfNode)
93154
}
94155
}
95156
} else {
96157
// update case
97-
// Assumption: updated routes (values) for given node will not overlap with other nodes.
158+
// Assumption: updated routes (values) for given node will not overlap with other nodes
98159
// So only updating existing routes of that node.
99160
utils.AviLog.Debugf("key: %s, StaticRoutes before updation/deletion: [%v]", key, utils.Stringify(aviVrfNode.StaticRoutes))
100-
startIndex := nodeStaticRouteDetails.StartIndex
101161
lenNewNodeRoutes := len(nodeRoutes)
102162
diff := lenNewNodeRoutes - nodeStaticRouteDetails.Count
103163

104-
var staticRouteCopy []*models.StaticRoute
105-
copyTill := int(math.Min(float64(startIndex), float64(len(aviVrfNode.StaticRoutes))))
106-
staticRouteCopy = append(staticRouteCopy, aviVrfNode.StaticRoutes[:copyTill]...)
107-
164+
staticRouteCopy := GetStaticRoutesForOtherNodes(aviVrfNode, routeIdPrefix)
108165
staticRouteCopy = append(staticRouteCopy, nodeRoutes...)
109-
110-
copyFrom := int(math.Min(float64(startIndex+nodeStaticRouteDetails.Count), float64(len(aviVrfNode.StaticRoutes))))
111-
staticRouteCopy = append(staticRouteCopy, aviVrfNode.StaticRoutes[copyFrom:]...)
112-
113166
aviVrfNode.StaticRoutes = staticRouteCopy
114167

115168
//if diff is 0, there is no change in number of routes previously exist and newly created.
116169
if diff != 0 {
117-
updateNodeStaticRoutes(aviVrfNode, deleteFlag, nodeName, lenNewNodeRoutes, diff)
170+
//update all the routes of this node
171+
updateNodeStaticRoutes(aviVrfNode, deleteFlag, nodeName, lenNewNodeRoutes)
118172
}
119173
if lenNewNodeRoutes == 0 {
120-
processNodeStaticRouteAndNodeIdDeletion(nodeName, aviVrfNode)
174+
//delete all the routes of this node
175+
processNodeStaticRouteAndNodeDeletion(nodeName, aviVrfNode)
121176
}
122177
utils.AviLog.Debugf("key: %s, StaticRoutes after updation/deletion: [%v]", key, utils.Stringify(aviVrfNode.StaticRoutes))
123178
}
124179
} else {
125-
//delete case
180+
//delete flag is turned on and node is deleted
126181
utils.AviLog.Debugf("key: %s, StaticRoutes before deletion: [%v]", key, utils.Stringify(aviVrfNode.StaticRoutes))
127-
startIndex := nodeStaticRouteDetails.StartIndex
128-
count := nodeStaticRouteDetails.Count
129-
var staticRouteCopy []*models.StaticRoute
130-
updateNodeStaticRoutes(aviVrfNode, deleteFlag, nodeName, 0, -count)
131-
132-
copyTill := int(math.Min(float64(startIndex), float64(len(aviVrfNode.StaticRoutes))))
133-
staticRouteCopy = append(staticRouteCopy, aviVrfNode.StaticRoutes[:copyTill]...)
134-
135-
copyFrom := int(math.Min(float64(startIndex+nodeStaticRouteDetails.Count), float64(len(aviVrfNode.StaticRoutes))))
136-
staticRouteCopy = append(staticRouteCopy, aviVrfNode.StaticRoutes[copyFrom:]...)
137-
138-
aviVrfNode.StaticRoutes = staticRouteCopy
139-
processNodeStaticRouteAndNodeIdDeletion(nodeName, aviVrfNode)
182+
aviVrfNode.StaticRoutes = GetStaticRoutesForOtherNodes(aviVrfNode, nodeStaticRouteDetails.RouteIDPrefix)
183+
processNodeStaticRouteAndNodeDeletion(nodeName, aviVrfNode)
140184
utils.AviLog.Debugf("key: %s, StaticRoutes after deletion: [%v]", key, utils.Stringify(aviVrfNode.StaticRoutes))
141185
}
142186
aviVrfNode.CalculateCheckSum()
@@ -145,29 +189,25 @@ func (o *AviObjectGraph) BuildVRFGraph(key, vrfName, nodeName string, deleteFlag
145189
utils.AviLog.Debugf("key: %s, vrf node: [%v]", key, utils.Stringify(aviVrfNode))
146190
return nil
147191
}
148-
func processNodeStaticRouteAndNodeIdDeletion(nodeName string, aviVrfNode *AviVrfNode) {
192+
func processNodeStaticRouteAndNodeDeletion(nodeName string, aviVrfNode *AviVrfNode) {
149193
delete(aviVrfNode.NodeStaticRoutes, nodeName)
150-
for nodeId := len(aviVrfNode.NodeIds); nodeId > len(aviVrfNode.StaticRoutes); nodeId-- {
151-
delete(aviVrfNode.NodeIds, nodeId)
152-
}
153-
}
154-
func findFreeRouteId(routeIdList map[int]struct{}) int {
155-
for i := 1; i < math.MaxInt32; i++ {
156-
if _, ok := routeIdList[i]; !ok {
157-
return i
194+
nodesCopy := []string{}
195+
for _, node := range aviVrfNode.Nodes {
196+
if node != nodeName {
197+
nodesCopy = append(nodesCopy, node)
158198
}
159199
}
160-
return -1
200+
aviVrfNode.Nodes = nodesCopy
161201
}
162-
func updateNodeStaticRoutes(aviVrfNode *AviVrfNode, isDelete bool, nodeName string, lenNewNodeRoutes, diff int) {
202+
203+
func updateNodeStaticRoutes(aviVrfNode *AviVrfNode, isDelete bool, nodeNameToUpdate string, lenNewNodeRoutes int) {
163204
//get index of nodename in node array
164205
indexOfNodeUnderUpdation := -1
165206

166207
for i := 0; i < len(aviVrfNode.Nodes); i++ {
167-
if aviVrfNode.Nodes[i] == nodeName {
208+
if aviVrfNode.Nodes[i] == nodeNameToUpdate {
168209
indexOfNodeUnderUpdation = i
169210
if !isDelete {
170-
nodeNameToUpdate := aviVrfNode.Nodes[indexOfNodeUnderUpdation]
171211
nodeDetails := aviVrfNode.NodeStaticRoutes[nodeNameToUpdate]
172212
nodeDetails.Count = lenNewNodeRoutes
173213
aviVrfNode.NodeStaticRoutes[nodeNameToUpdate] = nodeDetails
@@ -176,32 +216,6 @@ func updateNodeStaticRoutes(aviVrfNode *AviVrfNode, isDelete bool, nodeName stri
176216
}
177217
}
178218
if indexOfNodeUnderUpdation != -1 {
179-
clusterName := lib.GetClusterName()
180-
//Change nodemap entries till index
181-
for nodeIndex := len(aviVrfNode.Nodes) - 1; nodeIndex > indexOfNodeUnderUpdation; nodeIndex-- {
182-
nodeNameToUpdate := aviVrfNode.Nodes[nodeIndex]
183-
nodeDetails := aviVrfNode.NodeStaticRoutes[nodeNameToUpdate]
184-
oldStartIndex := nodeDetails.StartIndex
185-
nodeDetails.StartIndex = nodeDetails.StartIndex + diff
186-
nodeDetails.routeID = nodeDetails.StartIndex + 1
187-
aviVrfNode.NodeStaticRoutes[nodeNameToUpdate] = nodeDetails
188-
newRouteId := nodeDetails.routeID
189-
var tempStartIndex int
190-
if isDelete {
191-
tempStartIndex = oldStartIndex
192-
} else {
193-
tempStartIndex = nodeDetails.StartIndex
194-
}
195-
for staticRouteIndex := tempStartIndex; staticRouteIndex < nodeDetails.Count+tempStartIndex; staticRouteIndex++ {
196-
newRouteName := clusterName + "-" + strconv.Itoa(newRouteId)
197-
if staticRouteIndex > (len(aviVrfNode.StaticRoutes)-1) || staticRouteIndex < 0 {
198-
utils.AviLog.Warnf("Some StaticRoutes could not be updated.")
199-
continue
200-
}
201-
aviVrfNode.StaticRoutes[staticRouteIndex].RouteID = &newRouteName
202-
newRouteId++
203-
}
204-
}
205219
// lenNewNodeRoutes will be zero if Node exists without any PodCidr/BloackAffinity attached to it.
206220
if isDelete || lenNewNodeRoutes == 0 {
207221
// now remove nodename from Nodes list
@@ -226,7 +240,7 @@ func findRoutePrefix(nodeRoutes, aviRoutes []*models.StaticRoute, key string) bo
226240
return false
227241
}
228242

229-
func (o *AviObjectGraph) addRouteForNode(node *v1.Node, vrfName string, routeid int, routeIdList map[int]struct{}) ([]*models.StaticRoute, error) {
243+
func (o *AviObjectGraph) addRouteForNode(node *v1.Node, vrfName string, routeIdPrefix string) ([]*models.StaticRoute, error) {
230244
var nodeIP, nodeIP6 string
231245
var nodeRoutes []*models.StaticRoute
232246

@@ -243,7 +257,7 @@ func (o *AviObjectGraph) addRouteForNode(node *v1.Node, vrfName string, routeid
243257
utils.AviLog.Errorf("Error in fetching Pod CIDR for %v: %s", node.ObjectMeta.Name, err.Error())
244258
return nil, errors.New("podcidr not found")
245259
}
246-
for _, podCIDR := range podCIDRs {
260+
for index, podCIDR := range podCIDRs {
247261
podCIDRAndMask := strings.Split(podCIDR, "/")
248262
if len(podCIDRAndMask) != 2 {
249263
utils.AviLog.Errorf("Error in splitting Pod CIDR for %v", node.ObjectMeta.Name)
@@ -273,7 +287,7 @@ func (o *AviObjectGraph) addRouteForNode(node *v1.Node, vrfName string, routeid
273287
continue
274288
}
275289
mask32 := int32(mask)
276-
routeIDString := clusterName + "-" + strconv.Itoa(routeid)
290+
routeIDString := clusterName + "-" + routeIdPrefix + "-" + strconv.Itoa(index)
277291
nodeRoute := models.StaticRoute{
278292
RouteID: &routeIDString,
279293
Prefix: &models.IPAddrPrefix{
@@ -291,8 +305,6 @@ func (o *AviObjectGraph) addRouteForNode(node *v1.Node, vrfName string, routeid
291305
}
292306

293307
nodeRoutes = append(nodeRoutes, &nodeRoute)
294-
routeIdList[routeid] = struct{}{}
295-
routeid++
296308
}
297309
return nodeRoutes, nil
298310
}

‎internal/nodes/dequeue_ingestion.go

+3-1
Original file line numberDiff line numberDiff line change
@@ -889,7 +889,9 @@ func processNodeObj(key, nodename string, sharedQueue *utils.WorkerQueue, fullsy
889889
utils.AviLog.Errorf("key: %s, msg: Error creating vrf graph: %v", key, err)
890890
return
891891
}
892-
892+
if !fullsync {
893+
aviModelGraph.CheckAndDeduplicateRecords(key)
894+
}
893895
ok := saveAviModel(model_name, aviModelGraph, key)
894896
if ok && !fullsync {
895897
PublishKeyToRestLayer(model_name, key, sharedQueue)

0 commit comments

Comments
 (0)