Skip to content
This repository was archived by the owner on Mar 26, 2025. It is now read-only.

Commit e1218b0

Browse files
amurarubaluchicken
amuraru
authored and
baluchicken
committed
Disable envoy ingress upstream circuit breaker
Each envoy pod in kafka ingress (when envoy ingress is enabled) defaults to max_connections=1024 per upstream broker This is limiting kafka ingress capacity in larger clusters Disabling the circuit breaking per: https://www.envoyproxy.io/docs/envoy/latest/faq/load_balancing/disable_circuit_breaking
1 parent e9da663 commit e1218b0

File tree

2 files changed

+149
-9
lines changed

2 files changed

+149
-9
lines changed

‎controllers/tests/kafkacluster_controller_envoy_test.go‎

Lines changed: 108 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,18 @@ func expectEnvoyConfigMap(kafkaCluster *v1beta1.KafkaCluster, eListenerTemplate
9292
portValue: 9901
9393
staticResources:
9494
clusters:
95-
- connectTimeout: 1s
95+
- circuitBreakers:
96+
thresholds:
97+
- maxConnections: 1000000000
98+
maxPendingRequests: 1000000000
99+
maxRequests: 1000000000
100+
maxRetries: 1000000000
101+
- maxConnections: 1000000000
102+
maxPendingRequests: 1000000000
103+
maxRequests: 1000000000
104+
maxRetries: 1000000000
105+
priority: HIGH
106+
connectTimeout: 1s
96107
loadAssignment:
97108
clusterName: broker-0
98109
endpoints:
@@ -104,7 +115,18 @@ staticResources:
104115
portValue: 9094
105116
name: broker-0
106117
type: STRICT_DNS
107-
- connectTimeout: 1s
118+
- circuitBreakers:
119+
thresholds:
120+
- maxConnections: 1000000000
121+
maxPendingRequests: 1000000000
122+
maxRequests: 1000000000
123+
maxRetries: 1000000000
124+
- maxConnections: 1000000000
125+
maxPendingRequests: 1000000000
126+
maxRequests: 1000000000
127+
maxRetries: 1000000000
128+
priority: HIGH
129+
connectTimeout: 1s
108130
loadAssignment:
109131
clusterName: broker-1
110132
endpoints:
@@ -116,7 +138,18 @@ staticResources:
116138
portValue: 9094
117139
name: broker-1
118140
type: STRICT_DNS
119-
- connectTimeout: 1s
141+
- circuitBreakers:
142+
thresholds:
143+
- maxConnections: 1000000000
144+
maxPendingRequests: 1000000000
145+
maxRequests: 1000000000
146+
maxRetries: 1000000000
147+
- maxConnections: 1000000000
148+
maxPendingRequests: 1000000000
149+
maxRequests: 1000000000
150+
maxRetries: 1000000000
151+
priority: HIGH
152+
connectTimeout: 1s
120153
loadAssignment:
121154
clusterName: broker-2
122155
endpoints:
@@ -128,7 +161,18 @@ staticResources:
128161
portValue: 9094
129162
name: broker-2
130163
type: STRICT_DNS
131-
- connectTimeout: 1s
164+
- circuitBreakers:
165+
thresholds:
166+
- maxConnections: 1000000000
167+
maxPendingRequests: 1000000000
168+
maxRequests: 1000000000
169+
maxRetries: 1000000000
170+
- maxConnections: 1000000000
171+
maxPendingRequests: 1000000000
172+
maxRequests: 1000000000
173+
maxRetries: 1000000000
174+
priority: HIGH
175+
connectTimeout: 1s
132176
loadAssignment:
133177
clusterName: all-brokers
134178
endpoints:
@@ -328,7 +372,18 @@ func expectEnvoyWithConfigAz1(kafkaCluster *v1beta1.KafkaCluster) {
328372
portValue: 9901
329373
staticResources:
330374
clusters:
331-
- connectTimeout: 1s
375+
- circuitBreakers:
376+
thresholds:
377+
- maxConnections: 1000000000
378+
maxPendingRequests: 1000000000
379+
maxRequests: 1000000000
380+
maxRetries: 1000000000
381+
- maxConnections: 1000000000
382+
maxPendingRequests: 1000000000
383+
maxRequests: 1000000000
384+
maxRetries: 1000000000
385+
priority: HIGH
386+
connectTimeout: 1s
332387
loadAssignment:
333388
clusterName: broker-0
334389
endpoints:
@@ -340,7 +395,18 @@ staticResources:
340395
portValue: 9094
341396
name: broker-0
342397
type: STRICT_DNS
343-
- connectTimeout: 1s
398+
- circuitBreakers:
399+
thresholds:
400+
- maxConnections: 1000000000
401+
maxPendingRequests: 1000000000
402+
maxRequests: 1000000000
403+
maxRetries: 1000000000
404+
- maxConnections: 1000000000
405+
maxPendingRequests: 1000000000
406+
maxRequests: 1000000000
407+
maxRetries: 1000000000
408+
priority: HIGH
409+
connectTimeout: 1s
344410
loadAssignment:
345411
clusterName: all-brokers
346412
endpoints:
@@ -453,7 +519,18 @@ func expectEnvoyWithConfigAz2(kafkaCluster *v1beta1.KafkaCluster) {
453519
portValue: 9901
454520
staticResources:
455521
clusters:
456-
- connectTimeout: 1s
522+
- circuitBreakers:
523+
thresholds:
524+
- maxConnections: 1000000000
525+
maxPendingRequests: 1000000000
526+
maxRequests: 1000000000
527+
maxRetries: 1000000000
528+
- maxConnections: 1000000000
529+
maxPendingRequests: 1000000000
530+
maxRequests: 1000000000
531+
maxRetries: 1000000000
532+
priority: HIGH
533+
connectTimeout: 1s
457534
loadAssignment:
458535
clusterName: broker-1
459536
endpoints:
@@ -465,7 +542,18 @@ staticResources:
465542
portValue: 9094
466543
name: broker-1
467544
type: STRICT_DNS
468-
- connectTimeout: 1s
545+
- circuitBreakers:
546+
thresholds:
547+
- maxConnections: 1000000000
548+
maxPendingRequests: 1000000000
549+
maxRequests: 1000000000
550+
maxRetries: 1000000000
551+
- maxConnections: 1000000000
552+
maxPendingRequests: 1000000000
553+
maxRequests: 1000000000
554+
maxRetries: 1000000000
555+
priority: HIGH
556+
connectTimeout: 1s
469557
loadAssignment:
470558
clusterName: broker-2
471559
endpoints:
@@ -477,7 +565,18 @@ staticResources:
477565
portValue: 9094
478566
name: broker-2
479567
type: STRICT_DNS
480-
- connectTimeout: 1s
568+
- circuitBreakers:
569+
thresholds:
570+
- maxConnections: 1000000000
571+
maxPendingRequests: 1000000000
572+
maxRequests: 1000000000
573+
maxRetries: 1000000000
574+
- maxConnections: 1000000000
575+
maxPendingRequests: 1000000000
576+
maxRequests: 1000000000
577+
maxRetries: 1000000000
578+
priority: HIGH
579+
connectTimeout: 1s
481580
loadAssignment:
482581
clusterName: all-brokers
483582
endpoints:

‎pkg/resources/envoy/configmap.go‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ import (
2929

3030
"github.com/golang/protobuf/ptypes"
3131
"github.com/golang/protobuf/ptypes/duration"
32+
"github.com/golang/protobuf/ptypes/wrappers"
3233
"google.golang.org/protobuf/encoding/protojson"
3334
corev1 "k8s.io/api/core/v1"
3435
"k8s.io/apimachinery/pkg/runtime"
@@ -144,6 +145,26 @@ func GenerateEnvoyConfig(kc *v1beta1.KafkaCluster, elistener v1beta1.ExternalLis
144145
ConnectTimeout: &duration.Duration{Seconds: 1},
145146
ClusterDiscoveryType: &envoycluster.Cluster_Type{Type: envoycluster.Cluster_STRICT_DNS},
146147
LbPolicy: envoycluster.Cluster_ROUND_ROBIN,
148+
// disable circuit breakingL:
149+
// https://www.envoyproxy.io/docs/envoy/latest/faq/load_balancing/disable_circuit_breaking
150+
CircuitBreakers: &envoycluster.CircuitBreakers{
151+
Thresholds: []*envoycluster.CircuitBreakers_Thresholds{
152+
{
153+
Priority: envoycore.RoutingPriority_DEFAULT,
154+
MaxConnections: &wrappers.UInt32Value{Value: 1_000_000_000},
155+
MaxPendingRequests: &wrappers.UInt32Value{Value: 1_000_000_000},
156+
MaxRequests: &wrappers.UInt32Value{Value: 1_000_000_000},
157+
MaxRetries: &wrappers.UInt32Value{Value: 1_000_000_000},
158+
},
159+
{
160+
Priority: envoycore.RoutingPriority_HIGH,
161+
MaxConnections: &wrappers.UInt32Value{Value: 1_000_000_000},
162+
MaxPendingRequests: &wrappers.UInt32Value{Value: 1_000_000_000},
163+
MaxRequests: &wrappers.UInt32Value{Value: 1_000_000_000},
164+
MaxRetries: &wrappers.UInt32Value{Value: 1_000_000_000},
165+
},
166+
},
167+
},
147168
LoadAssignment: &envoyendpoint.ClusterLoadAssignment{
148169
ClusterName: fmt.Sprintf("broker-%d", brokerId),
149170
Endpoints: []*envoyendpoint.LocalityLbEndpoints{{
@@ -213,6 +234,26 @@ func GenerateEnvoyConfig(kc *v1beta1.KafkaCluster, elistener v1beta1.ExternalLis
213234
ConnectTimeout: &duration.Duration{Seconds: 1},
214235
ClusterDiscoveryType: &envoycluster.Cluster_Type{Type: envoycluster.Cluster_STRICT_DNS},
215236
LbPolicy: envoycluster.Cluster_ROUND_ROBIN,
237+
// disable circuit breakingL:
238+
// https://www.envoyproxy.io/docs/envoy/latest/faq/load_balancing/disable_circuit_breaking
239+
CircuitBreakers: &envoycluster.CircuitBreakers{
240+
Thresholds: []*envoycluster.CircuitBreakers_Thresholds{
241+
{
242+
Priority: envoycore.RoutingPriority_DEFAULT,
243+
MaxConnections: &wrappers.UInt32Value{Value: 1_000_000_000},
244+
MaxPendingRequests: &wrappers.UInt32Value{Value: 1_000_000_000},
245+
MaxRequests: &wrappers.UInt32Value{Value: 1_000_000_000},
246+
MaxRetries: &wrappers.UInt32Value{Value: 1_000_000_000},
247+
},
248+
{
249+
Priority: envoycore.RoutingPriority_HIGH,
250+
MaxConnections: &wrappers.UInt32Value{Value: 1_000_000_000},
251+
MaxPendingRequests: &wrappers.UInt32Value{Value: 1_000_000_000},
252+
MaxRequests: &wrappers.UInt32Value{Value: 1_000_000_000},
253+
MaxRetries: &wrappers.UInt32Value{Value: 1_000_000_000},
254+
},
255+
},
256+
},
216257
LoadAssignment: &envoyendpoint.ClusterLoadAssignment{
217258
ClusterName: allBrokerEnvoyConfigName,
218259
Endpoints: []*envoyendpoint.LocalityLbEndpoints{{

0 commit comments

Comments
 (0)