add circuit breaker thresholds (#1171)

## Summary
Add circuit breaker thersholds.

## Related issues
-
[ENG-2476](https://linear.app/pomerium/issue/ENG-2476/ingress-controller-add-circuit-breaker-settings)


## Checklist

- [x] reference any related issues
- [x] updated docs
- [x] updated unit tests
- [ ] updated UPGRADING.md
- [x] add appropriate tag (`improvement` / `bug` / etc)
- [x] ready for review

Author: calebdoxsey

apis/ingress/v1/pomerium_types.go

@@ -337,6 +337,9 @@ type PomeriumSpec struct {
 
 	// DownstreamMTLS sets the <a href="https://www.pomerium.com/docs/reference/downstream-mtls-settings">Downstream MTLS Settings</a>.
 	DownstreamMTLS *DownstreamMTLS `json:"downstreamMtls,omitempty"`
+
+	// CircuitBreakerThresholds sets the circuit breaker thresholds settings.
+	CircuitBreakerThresholds *CircuitBreakerThresholds `json:"circuitBreakerThresholds,omitempty"`
 }
 
 // OTEL configures OpenTelemetry.
@@ -410,6 +413,40 @@ type Timeouts struct {
 	Idle *metav1.Duration `json:"idle,omitempty"`
 }
 
+// CircuitBreakerThresholds are the circuit breaker thresholds.
+type CircuitBreakerThresholds struct {
+	// MaxConnections sets the maximum number of connections that Envoy will
+	// make to the upstream cluster. If not specified, the default is 1024.
+	//
+	// +kubebuilder:validation:Optional
+	MaxConnections *uint32 `json:"maxConnections,omitempty"`
+	// MaxPendingRequests sets the maximum number of pending requests that
+	// Envoy will allow to the upstream cluster. If not specified, the
+	// default is 1024. This limit is applied as a connection limit for
+	// non-HTTP traffic.
+	//
+	// +kubebuilder:validation:Optional
+	MaxPendingRequests *uint32 `json:"maxPendingRequests,omitempty"`
+	// MaxRequests sets the maximum number of parallel requests that Envoy
+	// will make to the upstream cluster. If not specified, the default is
+	// 1024. This limit does not apply to non-HTTP traffic.
+	//
+	// +kubebuilder:validation:Optional
+	MaxRequests *uint32 `json:"maxRequests,omitempty"`
+	// MaxRetries sets the maximum number of parallel retries that Envoy
+	// will allow to the upstream cluster. If not specified, the default is 3.
+	//
+	// +kubebuilder:validation:Optional
+	MaxRetries *uint32 `json:"maxRetries,omitempty"`
+	// MaxConnectionPools sets the maximum number of connection pools per
+	// cluster that Envoy will concurrently support at once. If not specified,
+	// the default is unlimited. Set this for clusters which create a large
+	// number of connection pools.
+	//
+	// +kubebuilder:validation:Optional
+	MaxConnectionPools *uint32 `json:"maxConnectionPools"`
+}
+
 // ResourceStatus represents the outcome of the latest attempt to reconcile
 // relevant Kubernetes resource with Pomerium.
 type ResourceStatus struct {

apis/ingress/v1/zz_generated.deepcopy.go

@@ -103,6 +103,46 @@ spec:
                 items:
                   type: string
                 type: array
+              circuitBreakerThresholds:
+                description: CircuitBreakerThresholds sets the circuit breaker thresholds
+                  settings.
+                properties:
+                  maxConnectionPools:
+                    description: |-
+                      MaxConnectionPools sets the maximum number of connection pools per
+                      cluster that Envoy will concurrently support at once. If not specified,
+                      the default is unlimited. Set this for clusters which create a large
+                      number of connection pools.
+                    format: int32
+                    type: integer
+                  maxConnections:
+                    description: |-
+                      MaxConnections sets the maximum number of connections that Envoy will
+                      make to the upstream cluster. If not specified, the default is 1024.
+                    format: int32
+                    type: integer
+                  maxPendingRequests:
+                    description: |-
+                      MaxPendingRequests sets the maximum number of pending requests that
+                      Envoy will allow to the upstream cluster. If not specified, the
+                      default is 1024. This limit is applied as a connection limit for
+                      non-HTTP traffic.
+                    format: int32
+                    type: integer
+                  maxRequests:
+                    description: |-
+                      MaxRequests sets the maximum number of parallel requests that Envoy
+                      will make to the upstream cluster. If not specified, the default is
+                      1024. This limit does not apply to non-HTTP traffic.
+                    format: int32
+                    type: integer
+                  maxRetries:
+                    description: |-
+                      MaxRetries sets the maximum number of parallel retries that Envoy
+                      will allow to the upstream cluster. If not specified, the default is 3.
+                    format: int32
+                    type: integer
+                type: object
               codecType:
                 description: CodecType sets the <a href="https://www.pomerium.com/docs/reference/codec-type">Codec
                   Type</a>.

config/crd/bases/ingress.pomerium.io_pomerium.yaml

@@ -245,6 +245,46 @@ spec:
                 items:
                   type: string
                 type: array
+              circuitBreakerThresholds:
+                description: CircuitBreakerThresholds sets the circuit breaker thresholds
+                  settings.
+                properties:
+                  maxConnectionPools:
+                    description: |-
+                      MaxConnectionPools sets the maximum number of connection pools per
+                      cluster that Envoy will concurrently support at once. If not specified,
+                      the default is unlimited. Set this for clusters which create a large
+                      number of connection pools.
+                    format: int32
+                    type: integer
+                  maxConnections:
+                    description: |-
+                      MaxConnections sets the maximum number of connections that Envoy will
+                      make to the upstream cluster. If not specified, the default is 1024.
+                    format: int32
+                    type: integer
+                  maxPendingRequests:
+                    description: |-
+                      MaxPendingRequests sets the maximum number of pending requests that
+                      Envoy will allow to the upstream cluster. If not specified, the
+                      default is 1024. This limit is applied as a connection limit for
+                      non-HTTP traffic.
+                    format: int32
+                    type: integer
+                  maxRequests:
+                    description: |-
+                      MaxRequests sets the maximum number of parallel requests that Envoy
+                      will make to the upstream cluster. If not specified, the default is
+                      1024. This limit does not apply to non-HTTP traffic.
+                    format: int32
+                    type: integer
+                  maxRetries:
+                    description: |-
+                      MaxRetries sets the maximum number of parallel retries that Envoy
+                      will allow to the upstream cluster. If not specified, the default is 3.
+                    format: int32
+                    type: integer
+                type: object
               codecType:
                 description: CodecType sets the <a href="https://www.pomerium.com/docs/reference/codec-type">Codec
                   Type</a>.

deployment.yaml

@@ -16,7 +16,8 @@ require (
 	github.com/iancoleman/strcase v0.3.0
 	github.com/martinlindhe/base36 v1.1.1
 	github.com/open-policy-agent/opa v1.5.0
-	github.com/pomerium/pomerium v0.28.1-0.20250509170134-93b8c93daa0f
+	github.com/pomerium/csrf v1.7.0
+	github.com/pomerium/pomerium v0.28.1-0.20250616153839-5ac7ae9c26be
 	github.com/rs/zerolog v1.34.0
 	github.com/sergi/go-diff v1.3.1
 	github.com/spf13/cobra v1.9.1
@@ -52,6 +53,7 @@ require (
 	cloud.google.com/go/storage v1.53.0 // indirect
 	contrib.go.opencensus.io/exporter/prometheus v0.4.2 // indirect
 	github.com/CAFxX/httpcompression v0.0.9 // indirect
+	github.com/DataDog/zstd v1.5.6-0.20230824185856-869dae002e5e // indirect
 	github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.27.0 // indirect
 	github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.51.0 // indirect
 	github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.51.0 // indirect
@@ -85,6 +87,14 @@ require (
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/cloudflare/circl v1.6.1 // indirect
 	github.com/cncf/xds/go v0.0.0-20250121191232-2f005788dc42 // indirect
+	github.com/cockroachdb/crlib v0.0.0-20241015224233-894974b3ad94 // indirect
+	github.com/cockroachdb/errors v1.11.3 // indirect
+	github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce // indirect
+	github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect
+	github.com/cockroachdb/pebble/v2 v2.0.4 // indirect
+	github.com/cockroachdb/redact v1.1.5 // indirect
+	github.com/cockroachdb/swiss v0.0.0-20250304010804-34a2c6a59016 // indirect
+	github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect
 	github.com/coreos/go-oidc/v3 v3.14.1 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/emicklei/go-restful/v3 v3.12.0 // indirect
@@ -96,6 +106,7 @@ require (
 	github.com/fxamacker/cbor/v2 v2.7.0 // indirect
 	github.com/gabriel-vasile/mimetype v1.4.8 // indirect
 	github.com/gaissmai/bart v0.20.4 // indirect
+	github.com/getsentry/sentry-go v0.27.0 // indirect
 	github.com/go-chi/chi/v5 v5.2.1 // indirect
 	github.com/go-ini/ini v1.67.0 // indirect
 	github.com/go-jose/go-jose/v3 v3.0.4 // indirect
@@ -113,7 +124,7 @@ require (
 	github.com/gobwas/glob v0.2.3 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect
-	github.com/golang/snappy v0.0.4 // indirect
+	github.com/golang/snappy v0.0.5-0.20231225225746-43d5d4cd4e0e // indirect
 	github.com/google/btree v1.1.3 // indirect
 	github.com/google/cel-go v0.25.0 // indirect
 	github.com/google/gnostic-models v0.6.9 // indirect
@@ -138,6 +149,8 @@ require (
 	github.com/jxskiss/base62 v1.1.0 // indirect
 	github.com/klauspost/compress v1.18.0 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.10 // indirect
+	github.com/kr/pretty v0.3.1 // indirect
+	github.com/kr/text v0.2.0 // indirect
 	github.com/leodido/go-urn v1.4.0 // indirect
 	github.com/libdns/libdns v1.0.0-beta.1 // indirect
 	github.com/lufia/plan9stats v0.0.0-20240513124658-fba389f38bae // indirect
@@ -160,18 +173,18 @@ require (
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
-	github.com/pomerium/csrf v1.7.0 // indirect
 	github.com/pomerium/datasource v0.18.2-0.20221108160055-c6134b5ed524 // indirect
 	github.com/pomerium/envoy-custom v1.33.0 // indirect
 	github.com/pomerium/protoutil v0.0.0-20240813175624-47b7ac43ff46 // indirect
 	github.com/pomerium/webauthn v0.0.0-20240603205124-0428df511172 // indirect
 	github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
 	github.com/prometheus/client_golang v1.22.0 // indirect
 	github.com/prometheus/client_model v0.6.2 // indirect
-	github.com/prometheus/common v0.63.0 // indirect
+	github.com/prometheus/common v0.64.0 // indirect
 	github.com/prometheus/procfs v0.16.1 // indirect
 	github.com/prometheus/statsd_exporter v0.22.7 // indirect
 	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
+	github.com/rogpeppe/go-internal v1.13.1 // indirect
 	github.com/rs/cors v1.11.1 // indirect
 	github.com/sagikazarmark/locafero v0.7.0 // indirect
 	github.com/shirou/gopsutil/v3 v3.24.5 // indirect
@@ -206,22 +219,23 @@ require (
 	go.opentelemetry.io/contrib/propagators/b3 v1.35.0 // indirect
 	go.opentelemetry.io/contrib/propagators/jaeger v1.35.0 // indirect
 	go.opentelemetry.io/contrib/propagators/ot v1.35.0 // indirect
-	go.opentelemetry.io/otel v1.35.0 // indirect
+	go.opentelemetry.io/otel v1.36.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.35.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.35.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.35.0 // indirect
-	go.opentelemetry.io/otel/metric v1.35.0 // indirect
-	go.opentelemetry.io/otel/sdk v1.35.0 // indirect
-	go.opentelemetry.io/otel/sdk/metric v1.35.0 // indirect
-	go.opentelemetry.io/otel/trace v1.35.0 // indirect
+	go.opentelemetry.io/otel/exporters/prometheus v0.58.0 // indirect
+	go.opentelemetry.io/otel/metric v1.36.0 // indirect
+	go.opentelemetry.io/otel/sdk v1.36.0 // indirect
+	go.opentelemetry.io/otel/sdk/metric v1.36.0 // indirect
+	go.opentelemetry.io/otel/trace v1.36.0 // indirect
 	go.opentelemetry.io/proto/otlp v1.6.0 // indirect
 	go.uber.org/automaxprocs v1.6.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap/exp v0.3.0 // indirect
 	golang.org/x/crypto v0.38.0 // indirect
 	golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f // indirect
 	golang.org/x/mod v0.24.0 // indirect
-	golang.org/x/oauth2 v0.29.0 // indirect
+	golang.org/x/oauth2 v0.30.0 // indirect
 	golang.org/x/sys v0.33.0 // indirect
 	golang.org/x/term v0.32.0 // indirect
 	golang.org/x/text v0.25.0 // indirect