diff --git a/pkg/deployer/values.go b/pkg/deployer/values.go index 3c497ef7200..4a00f44588c 100644 --- a/pkg/deployer/values.go +++ b/pkg/deployer/values.go @@ -115,6 +115,7 @@ type HelmImage struct { type HelmService struct { Type *string `json:"type,omitempty"` ClusterIP *string `json:"clusterIP,omitempty"` + LoadBalancerIP *string `json:"loadBalancerIP,omitempty"` ExtraAnnotations map[string]string `json:"extraAnnotations,omitempty"` ExtraLabels map[string]string `json:"extraLabels,omitempty"` ExternalTrafficPolicy *string `json:"externalTrafficPolicy,omitempty"` diff --git a/pkg/deployer/values_helpers.go b/pkg/deployer/values_helpers.go index dc5e372680f..dd07575b744 100644 --- a/pkg/deployer/values_helpers.go +++ b/pkg/deployer/values_helpers.go @@ -1,7 +1,9 @@ package deployer import ( + "errors" "fmt" + "net/netip" "regexp" "sort" "strings" @@ -18,6 +20,14 @@ import ( "github.com/kgateway-dev/kgateway/v2/pkg/pluginsdk/ir" ) +var ( + // ErrMultipleAddresses is returned when multiple addresses are specified in Gateway.spec.addresses + ErrMultipleAddresses = errors.New("multiple addresses given, only one address is supported") + + // ErrNoValidIPAddress is returned when no valid IP address is found in Gateway.spec.addresses + ErrNoValidIPAddress = errors.New("IP address in Gateway.spec.addresses not valid") +) + // This file contains helper functions that generate helm values in the format needed // by the deployer. @@ -108,16 +118,59 @@ func GetServiceValues(svcConfig *kgateway.Service) *HelmService { // convert the service type enum to its string representation; // if type is not set, it will default to 0 ("ClusterIP") var svcType *string - if svcConfig.GetType() != nil { - svcType = ptr.To(string(*svcConfig.GetType())) + var clusterIP *string + var extraAnnotations map[string]string + var extraLabels map[string]string + var externalTrafficPolicy *string + + if svcConfig != nil { + if svcConfig.GetType() != nil { + svcType = ptr.To(string(*svcConfig.GetType())) + } + clusterIP = svcConfig.GetClusterIP() + extraAnnotations = svcConfig.GetExtraAnnotations() + extraLabels = svcConfig.GetExtraLabels() + externalTrafficPolicy = svcConfig.GetExternalTrafficPolicy() } + return &HelmService{ Type: svcType, - ClusterIP: svcConfig.GetClusterIP(), - ExtraAnnotations: svcConfig.GetExtraAnnotations(), - ExtraLabels: svcConfig.GetExtraLabels(), - ExternalTrafficPolicy: svcConfig.GetExternalTrafficPolicy(), + ClusterIP: clusterIP, + ExtraAnnotations: extraAnnotations, + ExtraLabels: extraLabels, + ExternalTrafficPolicy: externalTrafficPolicy, + } +} + +// SetLoadBalancerIPFromGateway extracts the IP address from Gateway.spec.addresses +// and sets it on the HelmService if the service type is LoadBalancer. +// Only sets the IP if exactly one valid IP address is found in Gateway.spec.addresses. +// Returns an error if more than one address is specified or no valid IP address is found. +func SetLoadBalancerIPFromGateway(gw *gwv1.Gateway, svc *HelmService) error { + // Only extract IP if service type is LoadBalancer + if svc.Type == nil || *svc.Type != string(corev1.ServiceTypeLoadBalancer) { + return nil } + + if len(gw.Spec.Addresses) == 0 { + return nil + } + + if len(gw.Spec.Addresses) > 1 { + return fmt.Errorf("%w: gateway %s/%s has %d addresses", ErrMultipleAddresses, gw.Namespace, gw.Name, len(gw.Spec.Addresses)) + } + + addr := gw.Spec.Addresses[0] + // IPAddressType or nil (defaults to IPAddressType per Gateway API spec) + if addr.Type == nil || *addr.Type == gwv1.IPAddressType { + // Validate IP format + if parsedIP, err := netip.ParseAddr(addr.Value); err == nil && parsedIP.IsValid() { + svc.LoadBalancerIP = &addr.Value + return nil + } + } + + return fmt.Errorf("%w: gateway %s/%s has no valid IP address", ErrNoValidIPAddress, gw.Namespace, gw.Name) } // Convert service account values from GatewayParameters into helm values to be used by the deployer. diff --git a/pkg/deployer/values_helpers_test.go b/pkg/deployer/values_helpers_test.go index 9760136e154..262906b9fe8 100644 --- a/pkg/deployer/values_helpers_test.go +++ b/pkg/deployer/values_helpers_test.go @@ -4,6 +4,10 @@ import ( "testing" "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + gwv1 "sigs.k8s.io/gateway-api/apis/v1" ) func TestComponentLogLevelsToString(t *testing.T) { @@ -56,3 +60,175 @@ func TestComponentLogLevelsToString(t *testing.T) { }) } } + +func TestSetLoadBalancerIPFromGateway(t *testing.T) { + tests := []struct { + name string + addresses []gwv1.GatewaySpecAddress + serviceType *string + wantIP *string + wantErr error + }{ + { + name: "single valid IPv4 address with LoadBalancer service", + addresses: []gwv1.GatewaySpecAddress{ + {Type: ptr.To(gwv1.IPAddressType), Value: "203.0.113.10"}, + }, + serviceType: ptr.To(string(corev1.ServiceTypeLoadBalancer)), + wantIP: ptr.To("203.0.113.10"), + wantErr: nil, + }, + { + name: "single valid IPv6 address with LoadBalancer service", + addresses: []gwv1.GatewaySpecAddress{ + {Type: ptr.To(gwv1.IPAddressType), Value: "2001:db8::1"}, + }, + serviceType: ptr.To(string(corev1.ServiceTypeLoadBalancer)), + wantIP: ptr.To("2001:db8::1"), + wantErr: nil, + }, + { + name: "nil address type defaults to IPAddressType", + addresses: []gwv1.GatewaySpecAddress{ + {Type: nil, Value: "192.0.2.1"}, + }, + serviceType: ptr.To(string(corev1.ServiceTypeLoadBalancer)), + wantIP: ptr.To("192.0.2.1"), + wantErr: nil, + }, + { + name: "empty addresses array with LoadBalancer service", + addresses: []gwv1.GatewaySpecAddress{}, + serviceType: ptr.To(string(corev1.ServiceTypeLoadBalancer)), + wantIP: nil, + wantErr: nil, + }, + { + name: "multiple valid IP addresses returns error", + addresses: []gwv1.GatewaySpecAddress{ + {Type: ptr.To(gwv1.IPAddressType), Value: "203.0.113.10"}, + {Type: ptr.To(gwv1.IPAddressType), Value: "203.0.113.11"}, + }, + serviceType: ptr.To(string(corev1.ServiceTypeLoadBalancer)), + wantIP: nil, + wantErr: ErrMultipleAddresses, + }, + { + name: "multiple addresses with mixed types returns error", + addresses: []gwv1.GatewaySpecAddress{ + {Type: ptr.To(gwv1.HostnameAddressType), Value: "example.com"}, + {Type: ptr.To(gwv1.IPAddressType), Value: "203.0.113.10"}, + }, + serviceType: ptr.To(string(corev1.ServiceTypeLoadBalancer)), + wantIP: nil, + wantErr: ErrMultipleAddresses, + }, + { + name: "single hostname address returns error", + addresses: []gwv1.GatewaySpecAddress{ + {Type: ptr.To(gwv1.HostnameAddressType), Value: "example.com"}, + }, + serviceType: ptr.To(string(corev1.ServiceTypeLoadBalancer)), + wantIP: nil, + wantErr: ErrNoValidIPAddress, + }, + { + name: "single invalid IP address returns error", + addresses: []gwv1.GatewaySpecAddress{ + {Type: ptr.To(gwv1.IPAddressType), Value: "not-an-ip"}, + }, + serviceType: ptr.To(string(corev1.ServiceTypeLoadBalancer)), + wantIP: nil, + wantErr: ErrNoValidIPAddress, + }, + { + name: "single invalid IP address format returns error", + addresses: []gwv1.GatewaySpecAddress{ + {Type: ptr.To(gwv1.IPAddressType), Value: "256.256.256.256"}, + }, + serviceType: ptr.To(string(corev1.ServiceTypeLoadBalancer)), + wantIP: nil, + wantErr: ErrNoValidIPAddress, + }, + { + name: "nil type with valid IP returns IP", + addresses: []gwv1.GatewaySpecAddress{ + {Type: nil, Value: "203.0.113.10"}, + }, + serviceType: ptr.To(string(corev1.ServiceTypeLoadBalancer)), + wantIP: ptr.To("203.0.113.10"), + wantErr: nil, + }, + { + name: "nil type with invalid IP returns error", + addresses: []gwv1.GatewaySpecAddress{ + {Type: nil, Value: "invalid"}, + }, + serviceType: ptr.To(string(corev1.ServiceTypeLoadBalancer)), + wantIP: nil, + wantErr: ErrNoValidIPAddress, + }, + { + name: "three addresses returns error", + addresses: []gwv1.GatewaySpecAddress{ + {Type: ptr.To(gwv1.IPAddressType), Value: "203.0.113.10"}, + {Type: ptr.To(gwv1.IPAddressType), Value: "203.0.113.11"}, + {Type: ptr.To(gwv1.IPAddressType), Value: "203.0.113.12"}, + }, + serviceType: ptr.To(string(corev1.ServiceTypeLoadBalancer)), + wantIP: nil, + wantErr: ErrMultipleAddresses, + }, + { + name: "valid IP with ClusterIP service does not set IP", + addresses: []gwv1.GatewaySpecAddress{ + {Type: ptr.To(gwv1.IPAddressType), Value: "203.0.113.10"}, + }, + serviceType: ptr.To(string(corev1.ServiceTypeClusterIP)), + wantIP: nil, + wantErr: nil, + }, + { + name: "valid IP with nil service type does not set IP", + addresses: []gwv1.GatewaySpecAddress{ + {Type: ptr.To(gwv1.IPAddressType), Value: "203.0.113.10"}, + }, + serviceType: nil, + wantIP: nil, + wantErr: nil, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gw := &gwv1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-gateway", + Namespace: "default", + }, + Spec: gwv1.GatewaySpec{ + Addresses: tt.addresses, + }, + } + + svc := &HelmService{ + Type: tt.serviceType, + } + + err := SetLoadBalancerIPFromGateway(gw, svc) + if tt.wantErr != nil { + assert.Error(t, err, "expected error") + assert.ErrorIs(t, err, tt.wantErr, "error type mismatch") + assert.Nil(t, svc.LoadBalancerIP, "expected nil IP when error occurs") + } else { + assert.NoError(t, err, "unexpected error") + if tt.wantIP == nil { + assert.Nil(t, svc.LoadBalancerIP, "expected nil but got %v", svc.LoadBalancerIP) + } else { + assert.NotNil(t, svc.LoadBalancerIP, "expected non-nil IP") + assert.Equal(t, *tt.wantIP, *svc.LoadBalancerIP, "IP address mismatch") + } + } + }) + } +} diff --git a/pkg/kgateway/deployer/gateway_parameters.go b/pkg/kgateway/deployer/gateway_parameters.go index 450d079a2ab..a93a6721402 100644 --- a/pkg/kgateway/deployer/gateway_parameters.go +++ b/pkg/kgateway/deployer/gateway_parameters.go @@ -478,6 +478,10 @@ func (k *kgatewayParameters) getValues(gw *gwv1.Gateway, gwParam *kgateway.Gatew // service values gateway.Service = deployer.GetServiceValues(svcConfig) + // Extract loadBalancerIP from Gateway.spec.addresses and set it on the service if service type is LoadBalancer + if err := deployer.SetLoadBalancerIPFromGateway(gw, gateway.Service); err != nil { + return nil, err + } // serviceaccount values gateway.ServiceAccount = deployer.GetServiceAccountValues(svcAccountConfig) // pod template values diff --git a/pkg/kgateway/helm/agentgateway/templates/service.yaml b/pkg/kgateway/helm/agentgateway/templates/service.yaml index 10e830fb4cb..2d790bf8255 100644 --- a/pkg/kgateway/helm/agentgateway/templates/service.yaml +++ b/pkg/kgateway/helm/agentgateway/templates/service.yaml @@ -21,6 +21,9 @@ spec: {{- with $gateway.service.clusterIP }} clusterIP: {{ . }} {{- end }} + {{- with $gateway.service.loadBalancerIP }} + loadBalancerIP: {{ . }} + {{- end }} ports: {{- range $p := $gateway.ports }} - name: {{ $p.name }} diff --git a/pkg/kgateway/helm/envoy/templates/service.yaml b/pkg/kgateway/helm/envoy/templates/service.yaml index 6f859d089cb..fd79847d432 100644 --- a/pkg/kgateway/helm/envoy/templates/service.yaml +++ b/pkg/kgateway/helm/envoy/templates/service.yaml @@ -21,6 +21,9 @@ spec: {{- with $gateway.service.clusterIP }} clusterIP: {{ . }} {{- end }} + {{- with $gateway.service.loadBalancerIP }} + loadBalancerIP: {{ . }} + {{- end }} ports: {{- range $p := $gateway.ports }} - name: {{ $p.name }} diff --git a/test/deployer/internal_helm_test.go b/test/deployer/internal_helm_test.go index 96e83ce4957..672f50bdc71 100644 --- a/test/deployer/internal_helm_test.go +++ b/test/deployer/internal_helm_test.go @@ -121,6 +121,10 @@ wIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQBtestcertdata Name: "envoy-infrastructure", InputFile: "envoy-infrastructure", }, + { + Name: "gateway with static IP address", + InputFile: "loadbalancer-static-ip", + }, { Name: "agentgateway-params-primary", InputFile: "agentgateway-params-primary", diff --git a/test/deployer/testdata/loadbalancer-static-ip-out.yaml b/test/deployer/testdata/loadbalancer-static-ip-out.yaml new file mode 100644 index 00000000000..45ea6c3e4f6 --- /dev/null +++ b/test/deployer/testdata/loadbalancer-static-ip-out.yaml @@ -0,0 +1,339 @@ +apiVersion: v1 +automountServiceAccountToken: false +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/instance: gw + app.kubernetes.io/managed-by: kgateway + app.kubernetes.io/name: gw + app.kubernetes.io/version: 1.0.0-ci1 + gateway.networking.k8s.io/gateway-class-name: kgateway + gateway.networking.k8s.io/gateway-name: gw + kgateway: kube-gateway + name: gw +--- +apiVersion: v1 +data: + envoy.yaml: | + admin: + address: + socket_address: { address: 127.0.0.1, port_value: 19000 } + layered_runtime: + layers: + - name: static_layer + static_layer: + envoy.restart_features.use_eds_cache_for_ads: true + - name: admin_layer + admin_layer: {} + node: + cluster: gw.default + metadata: + role: kgateway-kube-gateway-api~default~gw + static_resources: + listeners: + - name: readiness_listener + address: + socket_address: { address: 0.0.0.0, port_value: 8082 } + filter_chains: + - filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + stat_prefix: ingress_http + codec_type: AUTO + route_config: + name: main_route + virtual_hosts: + - name: local_service + domains: ["*"] + routes: + - match: + path: "/ready" + headers: + - name: ":method" + string_match: + exact: GET + route: + cluster: admin_port_cluster + http_filters: + - name: envoy.filters.http.health_check + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.health_check.v3.HealthCheck + pass_through_mode: false + headers: + - name: ":path" + string_match: + exact: "/envoy-hc" + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + - name: prometheus_listener + address: + socket_address: + address: 0.0.0.0 + port_value: 9091 + filter_chains: + - filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + codec_type: AUTO + stat_prefix: prometheus + route_config: + name: prometheus_route + virtual_hosts: + - name: prometheus_host + domains: + - "*" + routes: + - match: + path: "/ready" + headers: + - name: ":method" + string_match: + exact: GET + route: + cluster: admin_port_cluster + - match: + prefix: "/metrics" + headers: + - name: ":method" + string_match: + exact: GET + route: + prefix_rewrite: /stats/prometheus?usedonly + cluster: admin_port_cluster + - match: + prefix: "/stats" + headers: + - name: ":method" + string_match: + exact: GET + route: + prefix_rewrite: /stats + cluster: admin_port_cluster + http_filters: + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + clusters: + - name: xds_cluster + alt_stat_name: xds_cluster + connect_timeout: 5.000s + load_assignment: + cluster_name: xds_cluster + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: xds.cluster.local + port_value: 9977 + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions + explicit_http_config: + http2_protocol_options: {} + http_filters: + - name: transform + typed_config: + "@type": type.googleapis.com/envoy.api.v2.filter.http.FilterTransformations + transformations: + - match: + prefix: "/" + route_transformations: + request_transformation: + transformation_template: + headers: + authorization: {"text": 'Bearer {{ "{{ trim(data_source(\"token\")) -}}" }}'} + passthrough: {} + data_sources: + token: + filename: "/var/run/secrets/tokens/xds-token" + watched_directory: + path: "/var/run/secrets/tokens" + - name: envoy.filters.http.upstream_codec + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.upstream_codec.v3.UpstreamCodec + upstream_connection_options: + tcp_keepalive: + keepalive_time: 10 + cluster_type: + name: envoy.cluster.strict_dns + typed_config: + "@type": type.googleapis.com/envoy.extensions.clusters.dns.v3.DnsCluster + respect_dns_ttl: true + - name: admin_port_cluster + connect_timeout: 5.000s + type: STATIC + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: admin_port_cluster + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: 127.0.0.1 + port_value: 19000 + dynamic_resources: + ads_config: + transport_api_version: V3 + api_type: GRPC + rate_limit_settings: {} + grpc_services: + - envoy_grpc: + cluster_name: xds_cluster + cds_config: + resource_api_version: V3 + ads: {} + lds_config: + resource_api_version: V3 + ads: {} +kind: ConfigMap +metadata: + labels: + app.kubernetes.io/instance: gw + app.kubernetes.io/managed-by: kgateway + app.kubernetes.io/name: gw + app.kubernetes.io/version: 1.0.0-ci1 + gateway.networking.k8s.io/gateway-class-name: kgateway + gateway.networking.k8s.io/gateway-name: gw + kgateway: kube-gateway + name: gw +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/instance: gw + app.kubernetes.io/managed-by: kgateway + app.kubernetes.io/name: gw + app.kubernetes.io/version: 1.0.0-ci1 + gateway.networking.k8s.io/gateway-class-name: kgateway + gateway.networking.k8s.io/gateway-name: gw + kgateway: kube-gateway + name: gw +spec: + loadBalancerIP: 203.0.113.10 + ports: + - name: listener-8080 + port: 8080 + protocol: TCP + targetPort: 8080 + selector: + app.kubernetes.io/instance: gw + app.kubernetes.io/name: gw + gateway.networking.k8s.io/gateway-name: gw + type: LoadBalancer +status: + loadBalancer: {} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.kubernetes.io/instance: gw + app.kubernetes.io/managed-by: kgateway + app.kubernetes.io/name: gw + app.kubernetes.io/version: 1.0.0-ci1 + gateway.networking.k8s.io/gateway-class-name: kgateway + gateway.networking.k8s.io/gateway-name: gw + kgateway: kube-gateway + name: gw +spec: + selector: + matchLabels: + app.kubernetes.io/instance: gw + app.kubernetes.io/name: gw + gateway.networking.k8s.io/gateway-name: gw + strategy: {} + template: + metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: "9091" + prometheus.io/scrape: "true" + labels: + app.kubernetes.io/instance: gw + app.kubernetes.io/name: gw + gateway.networking.k8s.io/gateway-class-name: kgateway + gateway.networking.k8s.io/gateway-name: gw + kgateway: kube-gateway + spec: + containers: + - args: + - --disable-hot-restart + - --service-node + - $(POD_NAME).$(POD_NAMESPACE) + - --log-level + - info + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: ENVOY_UID + value: "0" + image: ghcr.io/envoy-wrapper:v2.1.0-dev + lifecycle: + preStop: + exec: + command: + - /bin/sh + - -c + - wget --post-data "" -O /dev/null 127.0.0.1:19000/healthcheck/fail; + sleep 10 + name: kgateway-proxy + ports: + - containerPort: 8080 + name: listener-8080 + protocol: TCP + - containerPort: 9091 + name: http-monitoring + readinessProbe: + httpGet: + path: /ready + port: 8082 + periodSeconds: 10 + resources: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 10101 + startupProbe: + failureThreshold: 60 + httpGet: + path: /ready + port: 8082 + periodSeconds: 1 + successThreshold: 1 + timeoutSeconds: 2 + volumeMounts: + - mountPath: /etc/envoy + name: envoy-config + - mountPath: /var/run/secrets/tokens + name: xds-token + readOnly: true + serviceAccountName: gw + terminationGracePeriodSeconds: 60 + volumes: + - name: xds-token + projected: + sources: + - serviceAccountToken: + audience: kgateway + expirationSeconds: 43200 + path: xds-token + - configMap: + name: gw + name: envoy-config +status: {} diff --git a/test/deployer/testdata/loadbalancer-static-ip.yaml b/test/deployer/testdata/loadbalancer-static-ip.yaml new file mode 100644 index 00000000000..e7e57e82053 --- /dev/null +++ b/test/deployer/testdata/loadbalancer-static-ip.yaml @@ -0,0 +1,26 @@ +apiVersion: gateway.networking.k8s.io/v1 +kind: GatewayClass +metadata: + name: kgateway +spec: + controllerName: kgateway.dev/kgateway + description: Standard class for managing Gateway API ingress traffic. +--- +kind: Gateway +apiVersion: gateway.networking.k8s.io/v1 +metadata: + name: gw + namespace: default +spec: + gatewayClassName: kgateway + addresses: + - type: IPAddress + value: 203.0.113.10 + listeners: + - protocol: HTTP + port: 8080 + name: http + allowedRoutes: + namespaces: + from: Same +---