Skip to content

Commit a1bb237

Browse files
oilbeateroilbeater
committed
fix: use external_ids to identify vpc policy routes and static routes
Signed-off-by: Mengxin Liu <liumengxinfly@gmail.com>
1 parent f6c95e8 commit a1bb237

File tree

6 files changed

+144
-59
lines changed

6 files changed

+144
-59
lines changed

‎pkg/controller/node.go‎

Lines changed: 17 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -207,9 +207,10 @@ func (c *Controller) handleAddNode(key string) error {
207207
match = fmt.Sprintf("ip%d.dst == %s", af, nodeIP)
208208
action = kubeovnv1.PolicyRouteActionReroute
209209
externalIDs = map[string]string{
210-
"vendor": util.CniTypeName,
211-
"node": node.Name,
212-
"address-family": strconv.Itoa(af),
210+
ovs.ExternalIDVendor: util.CniTypeName,
211+
ovs.ExternalIDController: "node",
212+
ovs.ExternalIDResourceName: node.Name,
213+
"address-family": strconv.Itoa(af),
213214
}
214215
)
215216
klog.Infof("add policy route for router: %s, match %s, action %s, nexthop %s, externalID %v", c.config.ClusterRouter, match, action, ip, externalIDs)
@@ -945,6 +946,8 @@ func (c *Controller) addNodeGatewayStaticRoute() error {
945946
NextHopIP: nextHop,
946947
RouteTable: util.MainRouteTable,
947948
},
949+
"node",
950+
"node-gateway",
948951
); err != nil {
949952
klog.Errorf("failed to add static route for node gw: %v", err)
950953
return err
@@ -1120,10 +1123,11 @@ func (c *Controller) addPolicyRouteForLocalDNSCacheOnNode(dnsIPs []string, nodeP
11201123

11211124
var (
11221125
externalIDs = map[string]string{
1123-
"vendor": util.CniTypeName,
1124-
"node": nodeName,
1125-
"address-family": strconv.Itoa(af),
1126-
"isLocalDnsCache": "true",
1126+
ovs.ExternalIDVendor: util.CniTypeName,
1127+
ovs.ExternalIDController: "node",
1128+
ovs.ExternalIDResourceName: nodeName,
1129+
"address-family": strconv.Itoa(af),
1130+
"isLocalDnsCache": "true",
11271131
}
11281132
pgAs = strings.ReplaceAll(fmt.Sprintf("%s_ip%d", nodePortName, af), "-", ".")
11291133
action = kubeovnv1.PolicyRouteActionReroute
@@ -1134,7 +1138,7 @@ func (c *Controller) addPolicyRouteForLocalDNSCacheOnNode(dnsIPs []string, nodeP
11341138
matches.Add(fmt.Sprintf("ip%d.src == $%s && ip%d.dst == %s", af, pgAs, af, ip))
11351139
}
11361140

1137-
policies, err := c.OVNNbClient.GetLogicalRouterPoliciesByExtID(c.config.ClusterRouter, "node", nodeName)
1141+
policies, err := c.OVNNbClient.GetLogicalRouterPoliciesByExtID(c.config.ClusterRouter, ovs.ExternalIDResourceName, nodeName)
11381142
if err != nil {
11391143
klog.Errorf("failed to list logical router policies with external-ids:node = %q: %v", nodeName, err)
11401144
return err
@@ -1178,10 +1182,11 @@ func (c *Controller) addPolicyRouteForLocalDNSCacheOnNode(dnsIPs []string, nodeP
11781182

11791183
func (c *Controller) deletePolicyRouteForLocalDNSCacheOnNode(nodeName string, af int) error {
11801184
policies, err := c.OVNNbClient.ListLogicalRouterPolicies(c.config.ClusterRouter, -1, map[string]string{
1181-
"vendor": util.CniTypeName,
1182-
"node": nodeName,
1183-
"address-family": strconv.Itoa(af),
1184-
"isLocalDnsCache": "true",
1185+
ovs.ExternalIDVendor: util.CniTypeName,
1186+
ovs.ExternalIDController: "node",
1187+
ovs.ExternalIDResourceName: nodeName,
1188+
"address-family": strconv.Itoa(af),
1189+
"isLocalDnsCache": "true",
11851190
}, true)
11861191
if err != nil {
11871192
klog.Errorf("failed to list logical router policies: %v", err)

‎pkg/controller/pod.go‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -867,8 +867,9 @@ func (c *Controller) reconcileRouteSubnets(pod *v1.Pod, needRoutePodNets []*kube
867867
NextHopIP: nextHop,
868868
},
869869
map[string]string{
870-
"vendor": util.CniTypeName,
871-
"subnet": subnet.Name,
870+
ovs.ExternalIDVendor: util.CniTypeName,
871+
ovs.ExternalIDController: "pod",
872+
ovs.ExternalIDResourceName: fmt.Sprintf("%s/%s", pod.Namespace, pod.Name),
872873
},
873874
); err != nil {
874875
klog.Errorf("failed to add policy route, %v", err)
@@ -934,8 +935,9 @@ func (c *Controller) reconcileRouteSubnets(pod *v1.Pod, needRoutePodNets []*kube
934935
NextHopIP: pod.Annotations[util.NorthGatewayAnnotation],
935936
},
936937
map[string]string{
937-
"vendor": util.CniTypeName,
938-
"subnet": subnet.Name,
938+
ovs.ExternalIDVendor: util.CniTypeName,
939+
ovs.ExternalIDController: "pod",
940+
ovs.ExternalIDResourceName: fmt.Sprintf("%s/%s", pod.Namespace, pod.Name),
939941
},
940942
); err != nil {
941943
klog.Errorf("failed to add policy route, %v", err)

‎pkg/controller/subnet.go‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2051,8 +2051,9 @@ func getIPSuffix(protocol string) string {
20512051

20522052
func buildPolicyRouteExternalIDs(subnetName string, extraIDs map[string]string) map[string]string {
20532053
externalIDs := map[string]string{
2054-
"vendor": util.CniTypeName,
2055-
"subnet": subnetName,
2054+
ovs.ExternalIDVendor: util.CniTypeName,
2055+
ovs.ExternalIDController: "subnet",
2056+
ovs.ExternalIDResourceName: subnetName,
20562057
}
20572058
maps.Copy(externalIDs, extraIDs)
20582059
return externalIDs
@@ -2498,6 +2499,8 @@ func (c *Controller) addCustomVPCStaticRouteForSubnet(subnet *kubeovnv1.Subnet)
24982499
CIDR: v4Cidr,
24992500
NextHopIP: v4Gw,
25002501
},
2502+
"subnet",
2503+
subnet.Name,
25012504
); err != nil {
25022505
klog.Errorf("failed to add static route, %v", err)
25032506
return err
@@ -2512,6 +2515,8 @@ func (c *Controller) addCustomVPCStaticRouteForSubnet(subnet *kubeovnv1.Subnet)
25122515
CIDR: v6Cidr,
25132516
NextHopIP: v6Gw,
25142517
},
2518+
"subnet",
2519+
subnet.Name,
25152520
); err != nil {
25162521
klog.Errorf("failed to add static route, %v", err)
25172522
return err

‎pkg/controller/vpc.go‎

Lines changed: 52 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import (
2424
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
2525

2626
kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1"
27+
"github.com/kubeovn/kube-ovn/pkg/ovs"
2728
"github.com/kubeovn/kube-ovn/pkg/ovsdb/ovnnb"
2829
"github.com/kubeovn/kube-ovn/pkg/util"
2930
)
@@ -289,7 +290,23 @@ func (c *Controller) handleAddOrUpdateVpc(key string) error {
289290
return err
290291
}
291292

292-
learnFromARPRequest := true
293+
learnFromARPRequest := vpc.Spec.EnableExternal
294+
if !learnFromARPRequest {
295+
for _, subnetName := range vpc.Status.Subnets {
296+
subnet, err := c.subnetsLister.Get(subnetName)
297+
if err != nil {
298+
if k8serrors.IsNotFound(err) {
299+
continue
300+
}
301+
klog.Errorf("failed to get subnet %s for vpc %s: %v", subnetName, key, err)
302+
return err
303+
}
304+
if subnet.Spec.Vlan != "" && subnet.Spec.U2OInterconnection {
305+
learnFromARPRequest = true
306+
break
307+
}
308+
}
309+
}
293310

294311
if err = c.createVpcRouter(key, learnFromARPRequest); err != nil {
295312
klog.Errorf("failed to create vpc router for vpc %s: %v", key, err)
@@ -319,15 +336,19 @@ func (c *Controller) handleAddOrUpdateVpc(key string) error {
319336
}
320337

321338
// handle static route
339+
staticRouteExternalIDs := map[string]string{
340+
ovs.ExternalIDVendor: util.CniTypeName,
341+
ovs.ExternalIDController: "vpc",
342+
ovs.ExternalIDResourceName: vpc.Name,
343+
}
344+
322345
var (
323346
staticExistedRoutes []*ovnnb.LogicalRouterStaticRoute
324347
staticTargetRoutes []*kubeovnv1.StaticRoute
325348
staticRouteMapping map[string][]*kubeovnv1.StaticRoute
326-
externalIDs = map[string]string{"vendor": util.CniTypeName}
327349
)
328350

329-
// only manage static routes which are kube-ovn managed, by filtering for vendor util.CniTypeName
330-
staticExistedRoutes, err = c.OVNNbClient.ListLogicalRouterStaticRoutes(vpc.Name, nil, nil, "", externalIDs)
351+
staticExistedRoutes, err = c.OVNNbClient.ListLogicalRouterStaticRoutes(vpc.Name, nil, nil, "", staticRouteExternalIDs)
331352
if err != nil {
332353
klog.Errorf("failed to get vpc %s static route list, %v", vpc.Name, err)
333354
return err
@@ -464,15 +485,15 @@ func (c *Controller) handleAddOrUpdateVpc(key string) error {
464485
if item.BfdID != "" {
465486
klog.Infof("vpc %s add static ecmp route: %+v", vpc.Name, item)
466487
if err = c.OVNNbClient.AddLogicalRouterStaticRoute(
467-
vpc.Name, item.RouteTable, convertPolicy(item.Policy), item.CIDR, &item.BfdID, externalIDs, item.NextHopIP,
488+
vpc.Name, item.RouteTable, convertPolicy(item.Policy), item.CIDR, &item.BfdID, staticRouteExternalIDs, item.NextHopIP,
468489
); err != nil {
469490
klog.Errorf("failed to add bfd static route to vpc %s , %v", vpc.Name, err)
470491
return err
471492
}
472493
} else {
473494
klog.Infof("vpc %s add static route: %+v", vpc.Name, item)
474495
if err = c.OVNNbClient.AddLogicalRouterStaticRoute(
475-
vpc.Name, item.RouteTable, convertPolicy(item.Policy), item.CIDR, nil, externalIDs, item.NextHopIP,
496+
vpc.Name, item.RouteTable, convertPolicy(item.Policy), item.CIDR, nil, staticRouteExternalIDs, item.NextHopIP,
476497
); err != nil {
477498
klog.Errorf("failed to add normal static route to vpc %s , %v", vpc.Name, err)
478499
return err
@@ -481,6 +502,12 @@ func (c *Controller) handleAddOrUpdateVpc(key string) error {
481502
}
482503

483504
// handle policy route
505+
policyExternalIDs := map[string]string{
506+
ovs.ExternalIDVendor: util.CniTypeName,
507+
ovs.ExternalIDController: "vpc",
508+
ovs.ExternalIDResourceName: vpc.Name,
509+
}
510+
484511
var (
485512
policyRouteExisted, policyRouteNeedDel, policyRouteNeedAdd []*kubeovnv1.PolicyRoute
486513
policyRouteLogical []*ovnnb.LogicalRouterPolicy
@@ -493,33 +520,30 @@ func (c *Controller) handleAddOrUpdateVpc(key string) error {
493520
policyRouteNeedDel, policyRouteNeedAdd = diffPolicyRouteWithExisted(policyRouteExisted, vpc.Spec.PolicyRoutes)
494521
} else {
495522
if vpc.Spec.PolicyRoutes == nil {
496-
// do not clean default vpc policy routes
497-
if err = c.OVNNbClient.ClearLogicalRouterPolicy(vpc.Name); err != nil {
498-
klog.Errorf("clean all vpc %s policy route failed, %v", vpc.Name, err)
523+
// only clean vpc spec managed policy routes, not those created by subnet or egress gateway controllers
524+
if err = c.OVNNbClient.DeleteLogicalRouterPolicies(vpc.Name, -1, policyExternalIDs); err != nil {
525+
klog.Errorf("clean vpc %s spec policy routes failed, %v", vpc.Name, err)
499526
return err
500527
}
501528
} else {
502-
policyRouteLogical, err = c.OVNNbClient.ListLogicalRouterPolicies(vpc.Name, -1, nil, true)
529+
policyRouteLogical, err = c.OVNNbClient.ListLogicalRouterPolicies(vpc.Name, -1, policyExternalIDs, false)
503530
if err != nil {
504531
klog.Errorf("failed to get vpc %s policy route list, %v", vpc.Name, err)
505532
return err
506533
}
507-
// diff vpc policy route
508534
policyRouteNeedDel, policyRouteNeedAdd = diffPolicyRouteWithLogical(policyRouteLogical, vpc.Spec.PolicyRoutes)
509535
}
510536
}
511-
// delete policies non-exist
512537
for _, item := range policyRouteNeedDel {
513538
klog.Infof("delete policy route for router: %s, priority: %d, match %s", vpc.Name, item.Priority, item.Match)
514539
if err = c.OVNNbClient.DeleteLogicalRouterPolicy(vpc.Name, item.Priority, item.Match); err != nil {
515540
klog.Errorf("del vpc %s policy route failed, %v", vpc.Name, err)
516541
return err
517542
}
518543
}
519-
// add new policies
520544
for _, item := range policyRouteNeedAdd {
521-
klog.Infof("add policy route for router: %s, match %s, action %s, nexthop %s, externalID %v", c.config.ClusterRouter, item.Match, string(item.Action), item.NextHopIP, externalIDs)
522-
if err = c.OVNNbClient.AddLogicalRouterPolicy(vpc.Name, item.Priority, item.Match, string(item.Action), []string{item.NextHopIP}, nil, externalIDs); err != nil {
545+
klog.Infof("add policy route for router: %s, match %s, action %s, nexthop %s, externalID %v", vpc.Name, item.Match, string(item.Action), item.NextHopIP, policyExternalIDs)
546+
if err = c.OVNNbClient.AddLogicalRouterPolicy(vpc.Name, item.Priority, item.Match, string(item.Action), []string{item.NextHopIP}, nil, policyExternalIDs); err != nil {
523547
klog.Errorf("add policy route to vpc %s failed, %v", vpc.Name, err)
524548
return err
525549
}
@@ -580,7 +604,7 @@ func (c *Controller) handleAddOrUpdateVpc(key string) error {
580604
if subnet.Spec.Vpc == key {
581605
// Accelerate subnet update when vpc config is updated.
582606
// In case VPC not set namespaces, subnet will backoff and may take long time to back to ready.
583-
if subnet.Status.IsNotReady() || subnet.Spec.U2OInterconnection {
607+
if subnet.Status.IsNotReady() {
584608
c.addOrUpdateSubnetQueue.Add(subnet.Name)
585609
}
586610
if vpc.Name != util.DefaultVpc && vpc.Spec.EnableBfd && subnet.Spec.EnableEcmp {
@@ -936,22 +960,26 @@ func (c *Controller) batchDeletePolicyRouteFromVpc(name string, policies []*kube
936960
return nil
937961
}
938962

939-
func (c *Controller) addStaticRouteToVpc(name string, route *kubeovnv1.StaticRoute) error {
940-
externalIDs := map[string]string{"vendor": util.CniTypeName}
963+
func (c *Controller) addStaticRouteToVpc(vpcName string, route *kubeovnv1.StaticRoute, controller, resourceName string) error {
964+
externalIDs := map[string]string{
965+
ovs.ExternalIDVendor: util.CniTypeName,
966+
ovs.ExternalIDController: controller,
967+
ovs.ExternalIDResourceName: resourceName,
968+
}
941969
if route.BfdID != "" {
942-
klog.Infof("vpc %s add static ecmp route: %+v", name, route)
970+
klog.Infof("vpc %s add static ecmp route: %+v", vpcName, route)
943971
if err := c.OVNNbClient.AddLogicalRouterStaticRoute(
944-
name, route.RouteTable, convertPolicy(route.Policy), route.CIDR, &route.BfdID, externalIDs, route.NextHopIP,
972+
vpcName, route.RouteTable, convertPolicy(route.Policy), route.CIDR, &route.BfdID, externalIDs, route.NextHopIP,
945973
); err != nil {
946-
klog.Errorf("failed to add bfd static route to vpc %s , %v", name, err)
974+
klog.Errorf("failed to add bfd static route to vpc %s , %v", vpcName, err)
947975
return err
948976
}
949977
} else {
950-
klog.Infof("vpc %s add static route: %+v", name, route)
978+
klog.Infof("vpc %s add static route: %+v", vpcName, route)
951979
if err := c.OVNNbClient.AddLogicalRouterStaticRoute(
952-
name, route.RouteTable, convertPolicy(route.Policy), route.CIDR, nil, externalIDs, route.NextHopIP,
980+
vpcName, route.RouteTable, convertPolicy(route.Policy), route.CIDR, nil, externalIDs, route.NextHopIP,
953981
); err != nil {
954-
klog.Errorf("failed to add normal static route to vpc %s , %v", name, err)
982+
klog.Errorf("failed to add normal static route to vpc %s , %v", vpcName, err)
955983
return err
956984
}
957985
}

0 commit comments

Comments
 (0)