Merging Master to Kraft and Reverting Rack Removal

.github/actions/kind-create/action.yaml

@@ -45,3 +45,12 @@ runs:
       chmod 600 $kubeconfig_path
       echo "kubeconfig=$(echo $kubeconfig_path)" >> $GITHUB_OUTPUT
     shell: bash
+
+  - name: Install cloud-provider-kind
+    id: cloud-provider-kind
+    run: |
+      echo "Install cloud-provider-kind"
+      go install sigs.k8s.io/cloud-provider-kind@latest
+      kubectl label node e2e-kind-control-plane node.kubernetes.io/exclude-from-external-load-balancers- 
+      ~/go/bin/cloud-provider-kind &
+    shell: bash

.github/workflows/build-push-kafka-docker.yml

@@ -24,18 +24,21 @@ jobs:
           echo ::set-output name=version::${VERSION}
           echo ::set-output name=tags::${TAGS}
           echo ::set-output name=created::$(date -u +'%Y-%m-%dT%H:%M:%SZ')
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v3
       - name: Login to DockerHub
         if: startsWith(github.ref, 'refs/tags/')
         uses: docker/login-action@v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       - name: Build and push
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v5
         with:
           context: docker/kafka
+          platforms: linux/amd64,linux/arm64
           push: ${{ startsWith(github.ref, 'refs/tags/') }}
           tags: ${{ steps.prep.outputs.tags }}
           labels: |

.github/workflows/codeql-analysis.yml

@@ -43,7 +43,7 @@ jobs:
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v1
+      uses: github/codeql-action/init@v2
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -54,7 +54,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v1
+      uses: github/codeql-action/autobuild@v2
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl
@@ -68,4 +68,4 @@ jobs:
     #   make release
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v1
+      uses: github/codeql-action/analyze@v2

.github/workflows/e2e-test.yaml

@@ -18,6 +18,12 @@ jobs:
         uses: actions/setup-go@v4
         with:
           go-version: 1.21
+
+      # Enable Tmate Session if you'd like to Debut the E2E Kind Cluster
+      # - name: Setup tmate session
+      #   uses: mxschmitt/action-tmate@v3
+      #   with:
+      #     detached: true
 
       - name: Checkout code
         uses: actions/checkout@v4

.github/workflows/helm.yml

@@ -33,7 +33,7 @@ jobs:
 
       - name: Add Helm repositories
         run: |
-          helm repo add banzaicloud-stable "https://kubernetes-charts.banzaicloud.com"
+          # helm repo add banzaicloud-stable "https://kubernetes-charts.banzaicloud.com"
           helm repo add incubator "https://charts.helm.sh/incubator"
           helm repo add stable "https://charts.helm.sh/stable"
 

Makefile

@@ -156,6 +156,20 @@ docker-build: ## Build the operator docker image.
 docker-push: ## Push the operator docker image.
 	docker push ${IMG}
 
+# PLATFORMS defines the target platforms for the manager image be built to provide support to multiple
+# architectures. (i.e. make docker-buildx IMG=myregistry/mypoperator:0.0.1). To use this option you need to:
+# - be able to use docker buildx. More info: https://docs.docker.com/build/buildx/
+# - have enabled BuildKit. More info: https://docs.docker.com/develop/develop-images/build_enhancements/
+# - be able to push the image to your registry (i.e. if you do not set a valid value via IMG=<myregistry/image:<tag>> then the export will fail)
+# To adequately provide solutions that are compatible with multiple platforms, you should consider using this option.
+PLATFORMS ?= linux/arm64,linux/amd64
+.PHONY: docker-buildx
+docker-buildx: ## Build and push docker image for the manager for cross-platform support
+	- docker buildx create --name koperator-builder
+	docker buildx use koperator-builder
+	docker buildx build --push --platform=$(PLATFORMS) --tag ${IMG} -f Dockerfile .
+	- docker buildx rm koperator-builder
+
 bin/controller-gen: bin/controller-gen-$(CONTROLLER_GEN_VERSION) ## Symlink controller-gen-<version> into versionless controller-gen.
 	@ln -sf controller-gen-$(CONTROLLER_GEN_VERSION) bin/controller-gen
 

api/go.mod

@@ -7,7 +7,7 @@ require (
 	emperror.dev/errors v0.8.1
 	github.com/banzaicloud/istio-client-go v0.0.17
 	github.com/cert-manager/cert-manager v1.13.2
-	github.com/stretchr/testify v1.8.4
+	// github.com/stretchr/testify v1.8.4
 	golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa
 	gotest.tools v2.2.0+incompatible
 	k8s.io/api v0.28.4
@@ -16,7 +16,7 @@ require (
 )
 
 require (
-	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
+	// github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/go-logr/logr v1.3.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/google/go-cmp v0.5.9 // indirect
@@ -25,18 +25,26 @@ require (
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
-	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
+	// github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	golang.org/x/net v0.18.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
-	gopkg.in/yaml.v3 v3.0.1 // indirect
+	// gopkg.in/yaml.v3 v3.0.1 // indirect
 	k8s.io/klog/v2 v2.110.1 // indirect
 	k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
 )
 
+require github.com/stretchr/testify v1.8.4
+
+require (
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)
+
 // remove once https://github.com/cert-manager/cert-manager/issues/5953 is fixed
 replace github.com/Venafi/vcert/v4 => github.com/jetstack/vcert/v4 v4.9.6-0.20230127103832-3aa3dfd6613d

api/v1beta1/kafkacluster_types.go

@@ -178,7 +178,7 @@ type KafkaClusterSpec struct {
 	RollingUpgradeConfig        RollingUpgradeConfig    `json:"rollingUpgradeConfig"`
 	// Selector for broker pods that need to be recycled/reconciled
 	TaintedBrokersSelector *metav1.LabelSelector `json:"taintedBrokersSelector,omitempty"`
-	// +kubebuilder:validation:Enum=envoy;istioingress
+	// +kubebuilder:validation:Enum=envoy;contour;istioingress
 	// IngressController specifies the type of the ingress controller to be used for external listeners. The `istioingress` ingress controller type requires the `spec.istioControlPlane` field to be populated as well.
 	IngressController string `json:"ingressController,omitempty"`
 	// IstioControlPlane is a reference to the IstioControlPlane resource for envoy configuration. It must be specified if istio ingress is used.
@@ -190,13 +190,14 @@ type KafkaClusterSpec struct {
 	// when false, they will be kept so the Kafka cluster remains available for those Kafka clients which are still using the previous ingress setting.
 	// +kubebuilder:default=false
 	// +optional
-	RemoveUnusedIngressResources bool                `json:"removeUnusedIngressResources,omitempty"`
-	PropagateLabels              bool                `json:"propagateLabels,omitempty"`
-	CruiseControlConfig          CruiseControlConfig `json:"cruiseControlConfig"`
-	EnvoyConfig                  EnvoyConfig         `json:"envoyConfig,omitempty"`
-	MonitoringConfig             MonitoringConfig    `json:"monitoringConfig,omitempty"`
-	AlertManagerConfig           *AlertManagerConfig `json:"alertManagerConfig,omitempty"`
-	IstioIngressConfig           IstioIngressConfig  `json:"istioIngressConfig,omitempty"`
+	RemoveUnusedIngressResources bool                 `json:"removeUnusedIngressResources,omitempty"`
+	PropagateLabels              bool                 `json:"propagateLabels,omitempty"`
+	CruiseControlConfig          CruiseControlConfig  `json:"cruiseControlConfig"`
+	EnvoyConfig                  EnvoyConfig          `json:"envoyConfig,omitempty"`
+	ContourIngressConfig         ContourIngressConfig `json:"contourIngressConfig,omitempty"`
+	MonitoringConfig             MonitoringConfig     `json:"monitoringConfig,omitempty"`
+	AlertManagerConfig           *AlertManagerConfig  `json:"alertManagerConfig,omitempty"`
+	IstioIngressConfig           IstioIngressConfig   `json:"istioIngressConfig,omitempty"`
 	// Envs defines environment variables for Kafka broker Pods.
 	// Adding the "+" prefix to the name prepends the value to that environment variable instead of overwriting it.
 	// Add the "+" suffix to append.
@@ -239,15 +240,17 @@ type RollingUpgradeConfig struct {
 	// alerts with 'rollingupgrade'
 	FailureThreshold int `json:"failureThreshold"`
 
-	// ConcurrentBrokerRestartsAllowed controls how many brokers can be restarted in parallel during a rolling upgrade. If
+	// ConcurrentBrokerRestartCountPerRack controls how many brokers can be restarted in parallel during a rolling upgrade. If
 	// it is set to a value greater than 1, the operator will restart up to that amount of brokers in parallel, if the
 	// brokers are within the same rack (as specified by "broker.rack" in broker read-only configs). Since using Kafka broker
 	// racks spreads out the replicas, we know that restarting multiple brokers in the same rack will not cause more than
 	// 1/Nth of the replicas of a topic-partition to be unavailable at the same time, where N is the number of racks used.
 	// This is a safe way to speed up the rolling upgrade. Note that for the rack distribution explained above, Cruise Control
-	// requires `com.linkedin.kafka.cruisecontrol.analyzer.goals.RackAwareDistributionGoal` to be configured.
+	// requires `com.linkedin.kafka.cruisecontrol.analyzer.goals.RackAwareDistributionGoal` to be configured. Default value is 1.
+	// +kubebuilder:validation:Minimum=1
+	// +kubebuilder:default=1
 	// +optional
-	ConcurrentBrokerRestartsAllowed int `json:"concurrentBrokerRestartsAllowed,omitempty"`
+	ConcurrentBrokerRestartCountPerRack int `json:"concurrentBrokerRestartCountPerRack,omitempty"`
 }
 
 // DisruptionBudget defines the configuration for PodDisruptionBudget where the workload is managed by the kafka-operator
@@ -622,6 +625,10 @@ func (c IngressServiceSettings) GetServiceType() corev1.ServiceType {
 	return c.ServiceType
 }
 
+func (c ContourIngressConfig) GetBrokerFqdn(brokerId int32) string {
+	return strings.Replace(c.BrokerFQDNTemplate, "%id", strconv.Itoa(int(brokerId)), 1)
+}
+
 // Replace %id in brokerHostnameTemplate with actual broker id
 func (c EnvoyConfig) GetBrokerHostname(brokerId int32) string {
 	return strings.Replace(c.BrokerHostnameTemplate, "%id", strconv.Itoa(int(brokerId)), 1)
@@ -702,7 +709,7 @@ type ExternalListenerConfig struct {
 	// IngressControllerTargetPort defines the container port that the ingress controller uses for handling external traffic.
 	// If not defined, 29092 will be used as the default IngressControllerTargetPort value.
 	IngressControllerTargetPort *int32 `json:"ingressControllerTargetPort,omitempty"`
-	// +kubebuilder:validation:Enum=LoadBalancer;NodePort
+	// +kubebuilder:validation:Enum=LoadBalancer;NodePort;ClusterIP
 	// accessMethod defines the method which the external listener is exposed through.
 	// Two types are supported LoadBalancer and NodePort.
 	// The recommended and default is the LoadBalancer.
@@ -725,8 +732,16 @@ type Config struct {
 
 type IngressConfig struct {
 	IngressServiceSettings `json:",inline"`
-	IstioIngressConfig     *IstioIngressConfig `json:"istioIngressConfig,omitempty"`
-	EnvoyConfig            *EnvoyConfig        `json:"envoyConfig,omitempty"`
+	IstioIngressConfig     *IstioIngressConfig   `json:"istioIngressConfig,omitempty"`
+	EnvoyConfig            *EnvoyConfig          `json:"envoyConfig,omitempty"`
+	ContourIngressConfig   *ContourIngressConfig `json:"contourIngressConfig,omitempty"`
+}
+
+type ContourIngressConfig struct {
+	// TLS secret used for Contour IngressRoute resource
+	TLSSecretName string `json:"tlsSecretName"`
+	// Broker hostname template for Contour IngressRoute resource to generate broker hostnames.
+	BrokerFQDNTemplate string `json:"brokerFQDNTemplate"`
 }
 
 // InternalListenerConfig defines the internal listener config for Kafka
@@ -764,6 +779,9 @@ type CommonListenerSpec struct {
 	// At least one of the listeners should have this flag enabled
 	// +optional
 	UsedForInnerBrokerCommunication bool `json:"usedForInnerBrokerCommunication"`
+	// UsedForKafkaAdminCommunication allows for a different port to be returned when the koperator is checking for the port to use to check if kafka is operating.
+	// +optional
+	UsedForKafkaAdminCommunication bool `json:"usedForKafkaAdminCommunication,omitempty"`
 }
 
 func (c *CommonListenerSpec) GetServerSSLCertSecretName() string {