Skip to content

Commit d4a0756

Browse files
keithfzkeithfz
authored
feat(infra): Add rateLimitHpa support in EnvoyGateway API (#4983)
* add hpa support for rate limit in EnvoyGateway API Signed-off-by: keithfz <[email protected]>
1 parent c534621 commit d4a0756

File tree

11 files changed

+271
-2
lines changed

11 files changed

+271
-2
lines changed

‎api/v1alpha1/envoygateway_helpers.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -235,6 +235,10 @@ func (r *EnvoyGatewayProvider) GetEnvoyGatewayKubeProvider() *EnvoyGatewayKubern
235235

236236
r.Kubernetes.RateLimitDeployment.defaultKubernetesDeploymentSpec(DefaultRateLimitImage)
237237

238+
if r.Kubernetes.RateLimitHpa != nil {
239+
r.Kubernetes.RateLimitHpa.setDefault()
240+
}
241+
238242
if r.Kubernetes.ShutdownManager == nil {
239243
r.Kubernetes.ShutdownManager = &ShutdownManager{Image: ptr.To(DefaultShutdownManagerImage)}
240244
}

‎api/v1alpha1/envoygateway_types.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -202,6 +202,12 @@ type EnvoyGatewayKubernetesProvider struct {
202202
// +optional
203203
RateLimitDeployment *KubernetesDeploymentSpec `json:"rateLimitDeployment,omitempty"`
204204

205+
// RateLimitHpa defines the Horizontal Pod Autoscaler settings for Envoy ratelimit Deployment.
206+
// If the HPA is set, Replicas field from RateLimitDeployment will be ignored.
207+
//
208+
// +optional
209+
RateLimitHpa *KubernetesHorizontalPodAutoscalerSpec `json:"rateLimitHpa,omitempty"`
210+
205211
// Watch holds configuration of which input resources should be watched and reconciled.
206212
// +optional
207213
Watch *KubernetesWatchMode `json:"watch,omitempty"`

‎api/v1alpha1/zz_generated.deepcopy.go‎

Lines changed: 5 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎internal/infrastructure/kubernetes/ratelimit/resource_provider.go‎

Lines changed: 48 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ type ResourceRender struct {
4242

4343
rateLimit *egv1a1.RateLimit
4444
rateLimitDeployment *egv1a1.KubernetesDeploymentSpec
45+
rateLimitHpa *egv1a1.KubernetesHorizontalPodAutoscalerSpec
4546

4647
// ownerReferenceUID store the uid of its owner reference.
4748
ownerReferenceUID map[string]types.UID
@@ -53,6 +54,7 @@ func NewResourceRender(ns string, gateway *egv1a1.EnvoyGateway, ownerReferenceUI
5354
Namespace: ns,
5455
rateLimit: gateway.RateLimit,
5556
rateLimitDeployment: gateway.GetEnvoyGatewayProvider().GetEnvoyGatewayKubeProvider().RateLimitDeployment,
57+
rateLimitHpa: gateway.GetEnvoyGatewayProvider().GetEnvoyGatewayKubeProvider().RateLimitHpa,
5658
ownerReferenceUID: ownerReferenceUID,
5759
}
5860
}
@@ -288,6 +290,11 @@ func (r *ResourceRender) Deployment() (*appsv1.Deployment, error) {
288290
}
289291
}
290292

293+
// omit the deployment replicas if HPA is being set
294+
if r.rateLimitHpa != nil {
295+
deployment.Spec.Replicas = nil
296+
}
297+
291298
// apply merge patch to deployment
292299
var err error
293300
if deployment, err = r.rateLimitDeployment.ApplyMergePatch(deployment); err != nil {
@@ -309,11 +316,50 @@ func (r *ResourceRender) DaemonSet() (*appsv1.DaemonSet, error) {
309316

310317
// HorizontalPodAutoscalerSpec returns the `HorizontalPodAutoscaler` sets spec.
311318
func (r *ResourceRender) HorizontalPodAutoscalerSpec() (*egv1a1.KubernetesHorizontalPodAutoscalerSpec, error) {
312-
return nil, nil
319+
return r.rateLimitHpa, nil
313320
}
314321

315322
func (r *ResourceRender) HorizontalPodAutoscaler() (*autoscalingv2.HorizontalPodAutoscaler, error) {
316-
return nil, nil
323+
hpaConfig, err := r.HorizontalPodAutoscalerSpec()
324+
if hpaConfig == nil {
325+
return nil, err
326+
}
327+
328+
hpa := &autoscalingv2.HorizontalPodAutoscaler{
329+
TypeMeta: metav1.TypeMeta{
330+
APIVersion: "autoscaling/v2",
331+
Kind: "HorizontalPodAutoscaler",
332+
},
333+
ObjectMeta: metav1.ObjectMeta{
334+
Namespace: r.Namespace,
335+
Name: r.Name(),
336+
Labels: rateLimitLabels(),
337+
},
338+
Spec: autoscalingv2.HorizontalPodAutoscalerSpec{
339+
ScaleTargetRef: autoscalingv2.CrossVersionObjectReference{
340+
APIVersion: "apps/v1",
341+
Kind: "Deployment",
342+
},
343+
MinReplicas: hpaConfig.MinReplicas,
344+
MaxReplicas: ptr.Deref(hpaConfig.MaxReplicas, 1),
345+
Metrics: hpaConfig.Metrics,
346+
Behavior: hpaConfig.Behavior,
347+
},
348+
}
349+
350+
// set deployment target ref name
351+
deploymentConfig := r.rateLimitDeployment
352+
if deploymentConfig.Name != nil {
353+
hpa.Spec.ScaleTargetRef.Name = *deploymentConfig.Name
354+
} else {
355+
hpa.Spec.ScaleTargetRef.Name = r.Name()
356+
}
357+
358+
if hpa, err = hpaConfig.ApplyMergePatch(hpa); err != nil {
359+
return nil, err
360+
}
361+
362+
return hpa, nil
317363
}
318364

319365
// PodDisruptionBudgetSpec returns the `PodDisruptionBudget` sets spec.

‎internal/infrastructure/kubernetes/ratelimit/resource_provider_test.go‎

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ import (
1515
"github.com/stretchr/testify/assert"
1616
"github.com/stretchr/testify/require"
1717
appsv1 "k8s.io/api/apps/v1"
18+
autoscalingv2 "k8s.io/api/autoscaling/v2"
1819
corev1 "k8s.io/api/core/v1"
1920
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
2021
"k8s.io/apimachinery/pkg/api/resource"
@@ -765,6 +766,123 @@ func loadDeployment(caseName string) (*appsv1.Deployment, error) {
765766
return deployment, nil
766767
}
767768

769+
func TestHorizontalPodAutoscaler(t *testing.T) {
770+
cfg, err := config.New()
771+
require.NoError(t, err)
772+
cases := []struct {
773+
caseName string
774+
rateLimit *egv1a1.RateLimit
775+
rateLimitHpa *egv1a1.KubernetesHorizontalPodAutoscalerSpec
776+
rateLimitDeployment *egv1a1.KubernetesDeploymentSpec
777+
}{
778+
{
779+
caseName: "default",
780+
rateLimit: &egv1a1.RateLimit{
781+
Backend: egv1a1.RateLimitDatabaseBackend{
782+
Type: egv1a1.RedisBackendType,
783+
Redis: &egv1a1.RateLimitRedisSettings{
784+
URL: "redis.redis.svc:6379",
785+
},
786+
},
787+
},
788+
rateLimitHpa: &egv1a1.KubernetesHorizontalPodAutoscalerSpec{},
789+
},
790+
{
791+
caseName: "custom",
792+
rateLimit: &egv1a1.RateLimit{
793+
Backend: egv1a1.RateLimitDatabaseBackend{
794+
Type: egv1a1.RedisBackendType,
795+
Redis: &egv1a1.RateLimitRedisSettings{
796+
URL: "redis.redis.svc:6379",
797+
},
798+
},
799+
},
800+
rateLimitHpa: &egv1a1.KubernetesHorizontalPodAutoscalerSpec{
801+
MinReplicas: ptr.To[int32](5),
802+
MaxReplicas: ptr.To[int32](10),
803+
Metrics: []autoscalingv2.MetricSpec{
804+
{
805+
Resource: &autoscalingv2.ResourceMetricSource{
806+
Name: corev1.ResourceCPU,
807+
Target: autoscalingv2.MetricTarget{
808+
Type: autoscalingv2.UtilizationMetricType,
809+
AverageUtilization: ptr.To[int32](60),
810+
},
811+
},
812+
Type: autoscalingv2.ResourceMetricSourceType,
813+
},
814+
{
815+
Resource: &autoscalingv2.ResourceMetricSource{
816+
Name: corev1.ResourceMemory,
817+
Target: autoscalingv2.MetricTarget{
818+
Type: autoscalingv2.UtilizationMetricType,
819+
AverageUtilization: ptr.To[int32](70),
820+
},
821+
},
822+
Type: autoscalingv2.ResourceMetricSourceType,
823+
},
824+
},
825+
},
826+
},
827+
{
828+
caseName: "with-deployment-name",
829+
rateLimit: &egv1a1.RateLimit{
830+
Backend: egv1a1.RateLimitDatabaseBackend{
831+
Type: egv1a1.RedisBackendType,
832+
Redis: &egv1a1.RateLimitRedisSettings{
833+
URL: "redis.redis.svc:6379",
834+
},
835+
},
836+
},
837+
rateLimitHpa: &egv1a1.KubernetesHorizontalPodAutoscalerSpec{},
838+
rateLimitDeployment: &egv1a1.KubernetesDeploymentSpec{
839+
Name: ptr.To("foo"),
840+
},
841+
},
842+
}
843+
for _, tc := range cases {
844+
t.Run(tc.caseName, func(t *testing.T) {
845+
cfg.EnvoyGateway.RateLimit = tc.rateLimit
846+
847+
cfg.EnvoyGateway.Provider = &egv1a1.EnvoyGatewayProvider{
848+
Type: egv1a1.ProviderTypeKubernetes,
849+
Kubernetes: &egv1a1.EnvoyGatewayKubernetesProvider{
850+
RateLimitHpa: tc.rateLimitHpa,
851+
RateLimitDeployment: tc.rateLimitDeployment,
852+
},
853+
}
854+
r := NewResourceRender(cfg.Namespace, cfg.EnvoyGateway, ownerReferenceUID)
855+
hpa, err := r.HorizontalPodAutoscaler()
856+
require.NoError(t, err)
857+
858+
if *overrideTestData {
859+
hpaYAML, err := yaml.Marshal(hpa)
860+
require.NoError(t, err)
861+
// nolint:gosec
862+
err = os.WriteFile(fmt.Sprintf("testdata/hpa/%s.yaml", tc.caseName), hpaYAML, 0o644)
863+
require.NoError(t, err)
864+
return
865+
}
866+
867+
expected, err := loadHpa(tc.caseName)
868+
require.NoError(t, err)
869+
870+
assert.Equal(t, expected, hpa)
871+
})
872+
}
873+
}
874+
875+
func loadHpa(caseName string) (*autoscalingv2.HorizontalPodAutoscaler, error) {
876+
hpaYAML, err := os.ReadFile(fmt.Sprintf("testdata/hpa/%s.yaml", caseName))
877+
if err != nil {
878+
return nil, err
879+
}
880+
881+
hpa := &autoscalingv2.HorizontalPodAutoscaler{}
882+
_ = yaml.Unmarshal(hpaYAML, hpa)
883+
return hpa, nil
884+
}
885+
768886
func TestGetServiceURL(t *testing.T) {
769887
got := GetServiceURL("envoy-gateway-system", "example-cluster.local")
770888
assert.Equal(t, "grpc://envoy-ratelimit.envoy-gateway-system.svc.example-cluster.local:8081", got)

‎internal/infrastructure/kubernetes/ratelimit/testdata/hpa/custom.yaml‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
apiVersion: autoscaling/v2
2+
kind: HorizontalPodAutoscaler
3+
metadata:
4+
creationTimestamp: null
5+
labels:
6+
app.kubernetes.io/component: ratelimit
7+
app.kubernetes.io/managed-by: envoy-gateway
8+
app.kubernetes.io/name: envoy-ratelimit
9+
name: envoy-ratelimit
10+
namespace: envoy-gateway-system
11+
spec:
12+
maxReplicas: 10
13+
metrics:
14+
- resource:
15+
name: cpu
16+
target:
17+
averageUtilization: 60
18+
type: Utilization
19+
type: Resource
20+
- resource:
21+
name: memory
22+
target:
23+
averageUtilization: 70
24+
type: Utilization
25+
type: Resource
26+
minReplicas: 5
27+
scaleTargetRef:
28+
apiVersion: apps/v1
29+
kind: Deployment
30+
name: envoy-ratelimit
31+
status:
32+
currentMetrics: null
33+
desiredReplicas: 0

‎internal/infrastructure/kubernetes/ratelimit/testdata/hpa/default.yaml‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
apiVersion: autoscaling/v2
2+
kind: HorizontalPodAutoscaler
3+
metadata:
4+
creationTimestamp: null
5+
labels:
6+
app.kubernetes.io/component: ratelimit
7+
app.kubernetes.io/managed-by: envoy-gateway
8+
app.kubernetes.io/name: envoy-ratelimit
9+
name: envoy-ratelimit
10+
namespace: envoy-gateway-system
11+
spec:
12+
maxReplicas: 1
13+
metrics:
14+
- resource:
15+
name: cpu
16+
target:
17+
averageUtilization: 80
18+
type: Utilization
19+
type: Resource
20+
scaleTargetRef:
21+
apiVersion: apps/v1
22+
kind: Deployment
23+
name: envoy-ratelimit
24+
status:
25+
currentMetrics: null
26+
desiredReplicas: 0

‎internal/infrastructure/kubernetes/ratelimit/testdata/hpa/with-deployment-name.yaml‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
apiVersion: autoscaling/v2
2+
kind: HorizontalPodAutoscaler
3+
metadata:
4+
creationTimestamp: null
5+
labels:
6+
app.kubernetes.io/component: ratelimit
7+
app.kubernetes.io/managed-by: envoy-gateway
8+
app.kubernetes.io/name: envoy-ratelimit
9+
name: envoy-ratelimit
10+
namespace: envoy-gateway-system
11+
spec:
12+
maxReplicas: 1
13+
metrics:
14+
- resource:
15+
name: cpu
16+
target:
17+
averageUtilization: 80
18+
type: Utilization
19+
type: Resource
20+
scaleTargetRef:
21+
apiVersion: apps/v1
22+
kind: Deployment
23+
name: foo
24+
status:
25+
currentMetrics: null
26+
desiredReplicas: 0

‎release-notes/current.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ new features: |
1717
Added support for trusted CIDRs in the ClientIPDetectionSettings API
1818
Added support for sending attributes to external processor in EnvoyExtensionPolicy API
1919
Added support for patching EnvoyProxy.spec.provider.kubernetes.envoyHpa and EnvoyProxy.spec.provider.kubernetes.envoyPDB
20+
Added support for defining rateLimitHpa in EnvoyGateway API
2021
2122
# Fixes for bugs identified in previous versions.
2223
bug fixes: |

‎site/content/en/latest/api/extension_types.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1132,6 +1132,7 @@ _Appears in:_
11321132
| Field | Type | Required | Description |
11331133
| --- | --- | --- | --- |
11341134
| `rateLimitDeployment` | _[KubernetesDeploymentSpec](#kubernetesdeploymentspec)_ | false | RateLimitDeployment defines the desired state of the Envoy ratelimit deployment resource.<br />If unspecified, default settings for the managed Envoy ratelimit deployment resource<br />are applied. |
1135+
| `rateLimitHpa` | _[KubernetesHorizontalPodAutoscalerSpec](#kuberneteshorizontalpodautoscalerspec)_ | false | RateLimitHpa defines the Horizontal Pod Autoscaler settings for Envoy ratelimit Deployment.<br />If the HPA is set, Replicas field from RateLimitDeployment will be ignored. |
11351136
| `watch` | _[KubernetesWatchMode](#kuberneteswatchmode)_ | false | Watch holds configuration of which input resources should be watched and reconciled. |
11361137
| `deploy` | _[KubernetesDeployMode](#kubernetesdeploymode)_ | false | Deploy holds configuration of how output managed resources such as the Envoy Proxy data plane<br />should be deployed |
11371138
| `overwriteControlPlaneCerts` | _boolean_ | false | OverwriteControlPlaneCerts updates the secrets containing the control plane certs, when set. |
@@ -2521,6 +2522,7 @@ Envoy Gateway will revert back to this value every time reconciliation occurs.
25212522
See k8s.io.autoscaling.v2.HorizontalPodAutoScalerSpec.
25222523

25232524
_Appears in:_
2525+
- [EnvoyGatewayKubernetesProvider](#envoygatewaykubernetesprovider)
25242526
- [EnvoyProxyKubernetesProvider](#envoyproxykubernetesprovider)
25252527

25262528
| Field | Type | Required | Description |

0 commit comments

Comments
 (0)