diff --git a/pkg/flags/flags.go b/pkg/flags/flags.go index 679b7f0241..0fcfeddc7e 100644 --- a/pkg/flags/flags.go +++ b/pkg/flags/flags.go @@ -148,6 +148,7 @@ var F = struct { ManageL4LBLogging bool EnableNEGsForIngress bool EnableIPv6OnlyL4 bool + EnableL3NetLBOptIn bool L4ILBLegacyHeadStartTime time.Duration // =============================== @@ -361,6 +362,7 @@ L7 load balancing. CSV values accepted. Example: -node-port-ranges=80,8080,400-5 flag.BoolVar(&F.ReadOnlyMode, "read-only-controllers", false, "When enabled, this flag runs the IG, NEG, L4 ILB, and L4 NetLB controllers in a read-only mode. This prevents them from executing any mutating API calls (e.g., create, update, delete), allowing you to safely observe controller behavior without modifying resources. The Ingress controller is exempt from this mode.") flag.BoolVar(&F.EnableNEGsForIngress, "enable-negs-for-ingress", true, "Allow the NEG controller to create NEGs for Ingress services.") flag.BoolVar(&F.EnableIPv6OnlyL4, "enable-ipv6-only-l4", false, "Enables IPv6-only mode for the L4 ILB and NetLB controllers, disabling all IPv4-related logic and resource management.") + flag.BoolVar(&F.EnableL3NetLBOptIn, "enable-l3-netlb-opt-in", false, "Enables L3 opt in mode for NetLB, where it uses L3 Forwarding Rule and Backend Service regardless of the service spec.") flag.DurationVar(&F.L4ILBLegacyHeadStartTime, "prevent-legacy-race-l4-ilb", 0*time.Second, "Delay before processing new L4 ILB services without existing finalizers. This gives the legacy controller a head start to claim the service, preventing a race condition upon service creation.") } diff --git a/pkg/forwardingrules/equal.go b/pkg/forwardingrules/equal.go index b23a89640e..7c4beb5f1f 100644 --- a/pkg/forwardingrules/equal.go +++ b/pkg/forwardingrules/equal.go @@ -23,7 +23,7 @@ func EqualIPv4(fr1, fr2 *composite.ForwardingRule) (bool, error) { return false, fmt.Errorf("Equal(): failed to parse backend resource URL from wanted FR, err - %w", err) } return fr1.IPAddress == fr2.IPAddress && - strings.ToLower(fr1.IPProtocol) == strings.ToLower(fr2.IPProtocol) && + strings.EqualFold(fr1.IPProtocol, fr2.IPProtocol) && fr1.LoadBalancingScheme == fr2.LoadBalancingScheme && equalPorts(fr1.Ports, fr2.Ports, fr1.PortRange, fr2.PortRange) && utils.EqualCloudResourceIDs(id1, id2) && @@ -45,7 +45,7 @@ func EqualIPv6(fr1, fr2 *composite.ForwardingRule) (bool, error) { if err != nil { return false, fmt.Errorf("EqualIPv6(): failed to parse resource URL from FR, err - %w", err) } - return strings.ToLower(fr1.IPProtocol) == strings.ToLower(fr2.IPProtocol) && + return strings.EqualFold(fr1.IPProtocol, fr2.IPProtocol) && fr1.LoadBalancingScheme == fr2.LoadBalancingScheme && equalPorts(fr1.Ports, fr2.Ports, fr1.PortRange, fr2.PortRange) && utils.EqualCloudResourceIDs(id1, id2) && diff --git a/pkg/loadbalancers/forwarding_rules.go b/pkg/loadbalancers/forwarding_rules.go index 8142da0b2c..3875158bee 100644 --- a/pkg/loadbalancers/forwarding_rules.go +++ b/pkg/loadbalancers/forwarding_rules.go @@ -35,6 +35,7 @@ import ( "k8s.io/ingress-gce/pkg/events" "k8s.io/ingress-gce/pkg/flags" "k8s.io/ingress-gce/pkg/forwardingrules" + "k8s.io/ingress-gce/pkg/loadbalancers/l3" "k8s.io/ingress-gce/pkg/translator" "k8s.io/ingress-gce/pkg/utils" "k8s.io/ingress-gce/pkg/utils/namer" @@ -407,6 +408,12 @@ func (l4netlb *L4NetLB) ensureIPv4ForwardingRule(bsLink string) (*composite.Forw newFwdRule.PortRange = "" } + if l3.Wants(l4netlb.Service) { + newFwdRule.Ports, newFwdRule.PortRange = nil, "" + newFwdRule.AllPorts = true + newFwdRule.IPProtocol = forwardingrules.ProtocolL3 + } + if existingFwdRule != nil { if existingFwdRule.NetworkTier != newFwdRule.NetworkTier { resource := fmt.Sprintf("Forwarding rule (%v)", frName) diff --git a/pkg/loadbalancers/forwarding_rules_ipv6.go b/pkg/loadbalancers/forwarding_rules_ipv6.go index 22c60acddf..840d43c18d 100644 --- a/pkg/loadbalancers/forwarding_rules_ipv6.go +++ b/pkg/loadbalancers/forwarding_rules_ipv6.go @@ -33,6 +33,7 @@ import ( "k8s.io/ingress-gce/pkg/events" "k8s.io/ingress-gce/pkg/flags" "k8s.io/ingress-gce/pkg/forwardingrules" + "k8s.io/ingress-gce/pkg/loadbalancers/l3" "k8s.io/ingress-gce/pkg/utils" ) @@ -291,6 +292,11 @@ func (l4netlb *L4NetLB) buildExpectedIPv6ForwardingRule(bsLink, ipv6AddressToUse fr.Ports = utils.GetPorts(svcPorts) fr.PortRange = "" } + if l3.Wants(l4netlb.Service) { + fr.Ports, fr.PortRange = nil, "" + fr.AllPorts = true + fr.IPProtocol = forwardingrules.ProtocolL3 + } return fr, nil } diff --git a/pkg/loadbalancers/l3/wants.go b/pkg/loadbalancers/l3/wants.go new file mode 100644 index 0000000000..9a9321fca7 --- /dev/null +++ b/pkg/loadbalancers/l3/wants.go @@ -0,0 +1,30 @@ +package l3 + +import ( + corev1 "k8s.io/api/core/v1" + "k8s.io/ingress-gce/pkg/flags" +) + +// ExperimentAnnotation is the key for enabling experimental L3 support for NetLB services. +// Note that the controller must have the flag for the experiment also enabled. +const ExperimentAnnotation = "networking.gke.io/l3-experiment" + +// Wants determines if the service should use experimental L3 GCE resources. +// Controller must be run with experimental feature flag enabled for the annotation to take effect. +func Wants(svc *corev1.Service) bool { + if !flags.F.EnableL3NetLBOptIn { + return false + } + + acceptableValues := map[string]struct{}{ + "true": {}, "enabled": {}, "enable": {}, + "on": {}, "yes": {}, "True": {}, + } + + val, ok := svc.Annotations[ExperimentAnnotation] + if !ok { + return false + } + _, ok = acceptableValues[val] + return ok +} diff --git a/pkg/loadbalancers/l3/wants_test.go b/pkg/loadbalancers/l3/wants_test.go new file mode 100644 index 0000000000..2fea59af26 --- /dev/null +++ b/pkg/loadbalancers/l3/wants_test.go @@ -0,0 +1,77 @@ +package l3_test + +import ( + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/ingress-gce/pkg/flags" + "k8s.io/ingress-gce/pkg/loadbalancers/l3" +) + +func TestWants(t *testing.T) { + flagVal := flags.F.EnableL3NetLBOptIn + defer func() { + flags.F.EnableL3NetLBOptIn = flagVal + }() + testCases := []struct { + desc string + svc corev1.Service + want bool + }{ + { + desc: "empty", + svc: corev1.Service{}, + want: false, + }, + { + desc: "enabled", + svc: corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "networking.gke.io/l3-experiment": "enabled", + }, + }, + }, + want: true, + }, + { + desc: "true", + svc: corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "networking.gke.io/l3-experiment": "true", + }, + }, + }, + want: true, + }, + { + desc: "false", + svc: corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "networking.gke.io/l3-experiment": "false", + }, + }, + }, + want: false, + }, + } + for _, tC := range testCases { + t.Run(tC.desc, func(t *testing.T) { + flags.F.EnableL3NetLBOptIn = true + got := l3.Wants(&tC.svc) + if got != tC.want { + t.Errorf("WantsL3NetLB(%+v) = %v, want %v", tC.svc, got, tC.want) + } + }) + t.Run(tC.desc+" flag disabled", func(t *testing.T) { + flags.F.EnableL3NetLBOptIn = false + got := l3.Wants(&tC.svc) + if got != false { + t.Errorf("WantsL3NetLB(%+v) = %v, want %v", tC.svc, got, false) + } + }) + } +} diff --git a/pkg/loadbalancers/l4netlb.go b/pkg/loadbalancers/l4netlb.go index 4148db147a..43dead4f15 100644 --- a/pkg/loadbalancers/l4netlb.go +++ b/pkg/loadbalancers/l4netlb.go @@ -38,6 +38,7 @@ import ( "k8s.io/ingress-gce/pkg/forwardingrules" "k8s.io/ingress-gce/pkg/healthchecksl4" "k8s.io/ingress-gce/pkg/l4lb/metrics" + "k8s.io/ingress-gce/pkg/loadbalancers/l3" "k8s.io/ingress-gce/pkg/network" "k8s.io/ingress-gce/pkg/utils" "k8s.io/ingress-gce/pkg/utils/namer" @@ -353,11 +354,6 @@ func (l4netlb *L4NetLB) provideBackendService(syncResult *L4NetLBSyncResult, hcL bsName := l4netlb.namer.L4Backend(l4netlb.Service.Namespace, l4netlb.Service.Name) servicePorts := l4netlb.Service.Spec.Ports - protocol := string(utils.GetProtocol(servicePorts)) - if l4netlb.enableMixedProtocol { - protocol = backends.GetProtocol(servicePorts) - } - localityLbPolicy := l4netlb.determineBackendServiceLocalityPolicy() connectionTrackingPolicy := l4netlb.connectionTrackingPolicy() @@ -376,7 +372,7 @@ func (l4netlb *L4NetLB) provideBackendService(syncResult *L4NetLBSyncResult, hcL backendParams := backends.L4BackendServiceParams{ Name: bsName, HealthCheckLink: hcLink, - Protocol: protocol, + Protocol: l4netlb.backendProtocol(servicePorts), SessionAffinity: string(l4netlb.Service.Spec.SessionAffinity), Scheme: string(cloud.SchemeExternal), NamespacedName: l4netlb.NamespacedName, @@ -405,6 +401,17 @@ func (l4netlb *L4NetLB) provideBackendService(syncResult *L4NetLBSyncResult, hcL return bs.SelfLink } +func (l4netlb *L4NetLB) backendProtocol(servicePorts []corev1.ServicePort) string { + switch { + case l3.Wants(l4netlb.Service): + return backends.ProtocolL3 + case l4netlb.enableMixedProtocol: + return backends.GetProtocol(servicePorts) + default: + return string(utils.GetProtocol(servicePorts)) + } +} + func (l4netlb *L4NetLB) ensureDualStackResources(result *L4NetLBSyncResult, nodeNames []string, bsLink string) { if utils.NeedsIPv4(l4netlb.Service) { l4netlb.ensureIPv4Resources(result, nodeNames, bsLink) @@ -422,7 +429,7 @@ func (l4netlb *L4NetLB) ensureDualStackResources(result *L4NetLBSyncResult, node // - IPv4 Forwarding Rule // - IPv4 Firewall func (l4netlb *L4NetLB) ensureIPv4Resources(result *L4NetLBSyncResult, nodeNames []string, bsLink string) { - if l4netlb.enableMixedProtocol && forwardingrules.NeedsMixed(l4netlb.Service.Spec.Ports) { + if !l3.Wants(l4netlb.Service) && l4netlb.enableMixedProtocol && forwardingrules.NeedsMixed(l4netlb.Service.Spec.Ports) { l4netlb.ensureIPv4MixedResources(result, nodeNames, bsLink) return } @@ -436,11 +443,7 @@ func (l4netlb *L4NetLB) ensureIPv4Resources(result *L4NetLBSyncResult, nodeNames result.MetricsLegacyState.IsUserError = IsUserError(err) return } - if fr.IPProtocol == string(corev1.ProtocolTCP) { - result.Annotations[annotations.TCPForwardingRuleKey] = fr.Name - } else { - result.Annotations[annotations.UDPForwardingRuleKey] = fr.Name - } + result.Annotations[forwardingRuleAnnotationKey(fr)] = fr.Name result.MetricsLegacyState.IsManagedIP = ipAddrType == address.IPAddrManaged result.MetricsLegacyState.IsPremiumTier = fr.NetworkTier == cloud.NetworkTierPremium.ToGCEValue() @@ -453,6 +456,33 @@ func (l4netlb *L4NetLB) ensureIPv4Resources(result *L4NetLBSyncResult, nodeNames result.Status = utils.AddIPToLBStatus(result.Status, fr.IPAddress) } +func forwardingRuleAnnotationKey(fr *composite.ForwardingRule) string { + m := map[string]map[string]string{ + "IPV4": { + forwardingrules.ProtocolL3: annotations.L3ForwardingRuleKey, + forwardingrules.ProtocolTCP: annotations.TCPForwardingRuleKey, + forwardingrules.ProtocolUDP: annotations.UDPForwardingRuleKey, + }, + "IPV6": { + forwardingrules.ProtocolL3: annotations.L3ForwardingRuleIPv6Key, + forwardingrules.ProtocolTCP: annotations.TCPForwardingRuleIPv6Key, + forwardingrules.ProtocolUDP: annotations.UDPForwardingRuleIPv6Key, + }, + } + + version := fr.IpVersion + if version == "" { + version = "IPV4" + } + + protocol := fr.IPProtocol + if protocol == "" { + protocol = "UDP" + } + + return m[version][protocol] +} + func (l4netlb *L4NetLB) ensureIPv4MixedResources(result *L4NetLBSyncResult, nodeNames []string, bsLink string) { res, err := l4netlb.mixedManager.EnsureIPv4(bsLink) diff --git a/pkg/loadbalancers/l4netlbipv6.go b/pkg/loadbalancers/l4netlbipv6.go index bfeb8c1113..a9335ef21f 100644 --- a/pkg/loadbalancers/l4netlbipv6.go +++ b/pkg/loadbalancers/l4netlbipv6.go @@ -24,7 +24,6 @@ import ( "github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud" "github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud/meta" compute "google.golang.org/api/compute/v1" - corev1 "k8s.io/api/core/v1" "k8s.io/ingress-gce/pkg/annotations" "k8s.io/ingress-gce/pkg/firewalls" "k8s.io/ingress-gce/pkg/utils" @@ -50,11 +49,7 @@ func (l4netlb *L4NetLB) ensureIPv6Resources(syncResult *L4NetLBSyncResult, nodeN return } - if ipv6fr.IPProtocol == string(corev1.ProtocolTCP) { - syncResult.Annotations[annotations.TCPForwardingRuleIPv6Key] = ipv6fr.Name - } else { - syncResult.Annotations[annotations.UDPForwardingRuleIPv6Key] = ipv6fr.Name - } + syncResult.Annotations[forwardingRuleAnnotationKey(ipv6fr)] = ipv6fr.Name // Google Cloud creates ipv6 forwarding rules with IPAddress in CIDR form. We will take only first address trimmedIPv6Address := strings.Split(ipv6fr.IPAddress, "/")[0] @@ -120,6 +115,15 @@ func (l4netlb *L4NetLB) ensureIPv6NodesFirewall(ipAddress string, nodeNames []st svcPorts := l4netlb.Service.Spec.Ports portRanges := utils.GetServicePortRanges(svcPorts) protocol := utils.GetProtocol(svcPorts) + allowed := []*compute.FirewallAllowed{ + { + IPProtocol: string(protocol), + Ports: portRanges, + }, + } + if l4netlb.enableMixedProtocol { + allowed = firewalls.AllowedForService(svcPorts) + } fwLogger := l4netlb.svcLogger.WithValues("firewallName", firewallName) fwLogger.V(2).Info("Ensuring IPv6 nodes firewall for L4 NetLB Service", "ipAddress", ipAddress, "protocol", protocol, "len(nodeNames)", len(nodeNames), "portRanges", portRanges) @@ -135,12 +139,7 @@ func (l4netlb *L4NetLB) ensureIPv6NodesFirewall(ipAddress string, nodeNames []st } ipv6nodesFWRParams := firewalls.FirewallParams{ - Allowed: []*compute.FirewallAllowed{ - { - IPProtocol: string(protocol), - Ports: portRanges, - }, - }, + Allowed: allowed, SourceRanges: ipv6SourceRanges, DestinationRanges: []string{ipAddress}, Name: firewallName,