Add support for dedicated envoy deployment per brokerConfigGroup (#2)

* No LB on envoy

* Add support for dedicated envoy deployment per brokerConfigGroup

Use a different label for each envoy deployment

Add support to bring your own Load Balancer

LB will not be managed by Kafka-Operator.

HostnameOverride should be mandatory with BringYourOwnLB

Add external listeners per broker config group

Create Envoy if all config groups define an EL

Envoy ingress should be creted if there is a global
External Listener define, or if each broker config
group defines each own External Listener.
The broker config group External Listener will
always override the global External Listener.

Reconcile External Listeners only if needed.

Fix LoadBalancer overwrite for ExternalListener

LB address / IP should be used if HostnameOverride is not used

Cosmetic fixes

Co-authored-by: Adi Muraru <[email protected]>

Author: alungu

charts/kafka-operator/crds/operator-kafka-crd.yaml

@@ -500,6 +500,54 @@ spec:
                           type: string
                       type: object
                     type: array
+                  envoy:
+                    description: Custom properties for the dedicated envoy deployment
+                      for this broker configuration group.
+                      Applicable only if EnvoyConfig.dedicatedEnvoyPerBrokerGroup is
+                      set to true.
+                    properties:
+                      replicas:
+                        format: int32
+                        minimum: 1
+                        type: integer
+                    required:
+                      - replicas
+                    type: object
+                  listenersConfig:
+                    description: ListenersConfig defines the broker specific Kafka
+                     listener types. These will override the global ListenersConfig
+                    properties:
+                      externalListeners:
+                        items:
+                          description: ExternalListenerConfig defines the external listener
+                            config for Kafka
+                          properties:
+                            containerPort:
+                              format: int32
+                              type: integer
+                            externalStartingPort:
+                              format: int32
+                              type: integer
+                              description: The broker port is computed as the sum of the
+                                externalStartingPort and the broker id. For consistence,
+                                it is recommended that externalStartingPort has the same
+                                value across all the brokerConfigGroups.
+                            hostnameOverride:
+                              type: string
+                            name:
+                              type: string
+                            type:
+                              type: string
+                          required:
+                            - containerPort
+                            - externalStartingPort
+                            - name
+                            - type
+                          type: object
+                        type: array
+                    required:
+                      - externalListeners
+                    type: object
                 type: object
               type: object
             brokers:
@@ -1131,6 +1179,16 @@ spec:
                         type: string
                     type: object
                   type: array
+                useExistingLoadBalancer:
+                  description: Set this to true if you don't want the operator to create
+                    a LoadBalancer service. This is recommended if you want to use your
+                    own LoadBalancer.
+                  type: boolean
+                dedicatedEnvoyPerBrokerGroup:
+                  description: Set this to true if you don't want the operator to create
+                    one Envoy deployment for each BrokerConifgGrop. This is recommended
+                    if you want to keep the brokers and the envoy in the same group/rack.
+                  type: boolean
                 loadBalancerSourceRanges:
                   items:
                     type: string
@@ -1401,6 +1459,8 @@ spec:
                       externalStartingPort:
                         format: int32
                         type: integer
+                        description: The broker port is computed as the sum of the
+                          externalStartingPort and the broker id.
                       hostnameOverride:
                         type: string
                       name:

config/base/crds/kafka.banzaicloud.io_kafkaclusters.yaml

@@ -505,6 +505,54 @@ spec:
                           type: string
                       type: object
                     type: array
+                  envoy:
+                    description: Custom properties for the dedicated envoy deployment
+                      for this broker configuration group.
+                      Applicable only if EnvoyConfig.dedicatedEnvoyPerBrokerGroup is
+                      set to true.
+                    properties:
+                      replicas:
+                        format: int32
+                        minimum: 1
+                        type: integer
+                    required:
+                      - replicas
+                    type: object
+                  listenersConfig:
+                    description: ListenersConfig defines the broker specific Kafka
+                      listener types. These will override the global ListenersConfig
+                    properties:
+                      externalListeners:
+                        items:
+                          description: ExternalListenerConfig defines the external listener
+                            config for Kafka
+                          properties:
+                            containerPort:
+                              format: int32
+                              type: integer
+                            externalStartingPort:
+                              format: int32
+                              type: integer
+                              description: The broker port is computed as the sum of the
+                                externalStartingPort and the broker id. For consistence,
+                                it is recommended that externalStartingPort has the same
+                                value across all the brokerConfigGroups.
+                            hostnameOverride:
+                              type: string
+                            name:
+                              type: string
+                            type:
+                              type: string
+                          required:
+                            - containerPort
+                            - externalStartingPort
+                            - name
+                            - type
+                          type: object
+                        type: array
+                    required:
+                      - externalListeners
+                    type: object
                 type: object
               type: object
             brokers:
@@ -520,6 +568,14 @@ spec:
                         type: object
                       config:
                         type: string
+                      dedicatedEnvoyDeployment:
+                        properties:
+                          enabled:
+                            type: boolean
+                          replicas:
+                            format: int32
+                            type: integer
+                        type: object
                       image:
                         type: string
                       imagePullSecrets:
@@ -1126,6 +1182,11 @@ spec:
                   additionalProperties:
                     type: string
                   type: object
+                useExistingLBLoadBalancer:
+                  description: Set this to true if you don't want the operator to create
+                    a LoadBalancer service. This is recommended if you want to use your
+                    own LoadBalancer.
+                  type: boolean
                 image:
                   type: string
                 imagePullSecrets:
@@ -1513,6 +1574,8 @@ spec:
                       externalStartingPort:
                         format: int32
                         type: integer
+                        description: The broker port is computed as the sum of the
+                          externalStartingPort and the broker id.
                       hostnameOverride:
                         type: string
                       name: