feat: migrate Kafka to KRaft (#162)

* feat: kraft migration

Author: Dariocent

.gitignore

@@ -1,3 +1,4 @@
 bin/
 *.tmp.yaml
-dockerconfig.json
+dockerconfig.json
+.DS_Store

nebuly-platform/CHANGELOG.md

@@ -1,15 +1,26 @@
 # Changelog
 
-## v1.78.1
+## v1.79.0
 
 ### Features
-* Strimzi Kafka Operator now uses NodePools CR to manage brokers
+* Kafka now uses KRaft as consesus algorithm and deprecates Zookeeper
 
-## v1.78.0
+### Breaking changes
+* If using Kafka through Strimzi operator:
+  - The upgrade to this version **MUST** be done from v1.78.0
+  - First set in the values.yaml **kafka.kraft=disabled**, then **kafka.kraft=migration**
+  - and finally, after it is finished, remove it so it will default to **enabled**
+  
+## v1.78.1
 
 ### Fixes
 * Fix to broken SQL query in the backend
 
+## v1.78.0
+
+### Features
+* Strimzi Kafka Operator now uses NodePools CR to manage brokers
+
 ## v1.17.0
 
 ### Features

nebuly-platform/Chart.yaml

@@ -7,7 +7,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 1.78.1
+version: 1.79.0
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

nebuly-platform/README.md

@@ -1,6 +1,6 @@
 # Nebuly Platform
 
-![Version: 1.78.1](https://img.shields.io/badge/Version-1.78.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.0.0](https://img.shields.io/badge/AppVersion-1.0.0-informational?style=flat-square)
+![Version: 1.79.0](https://img.shields.io/badge/Version-1.79.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.0.0](https://img.shields.io/badge/AppVersion-1.0.0-informational?style=flat-square)
 
 Helm chart for installing Nebuly's Platform on Kubernetes.
 
@@ -509,11 +509,20 @@ The command removes all the Kubernetes components associated with the chart and
 | kafka.affinity | object | `{}` |  |
 | kafka.bootstrapServers | string | `""` | [external] Comma separated list of Kafka brokers. |
 | kafka.config."replica.selector.class" | string | `"org.apache.kafka.common.replica.RackAwareReplicaSelector"` |  |
+| kafka.controller.affinity | object | `{}` |  |
+| kafka.controller.replicas | int | `3` |  |
+| kafka.controller.resources.limits.memory | string | `"2048Mi"` |  |
+| kafka.controller.resources.requests.cpu | string | `"100m"` |  |
+| kafka.controller.resources.requests.memory | string | `"1024Mi"` |  |
+| kafka.controller.storage.deleteClaim | bool | `false` |  |
+| kafka.controller.storage.size | string | `"10Gi"` |  |
+| kafka.controller.storage.type | string | `"persistent-claim"` |  |
 | kafka.createTopics | bool | `true` | [external] If True, create the Kafka topics automatically if not present on the specified external Kafka cluster. |
 | kafka.existingSecret | object | - | [external] Use an existing secret for Kafka authentication. |
 | kafka.existingSecret.name | string | `""` | Name of the secret. Can be templated. |
 | kafka.existingSecret.sslCaCertKey | string | `""` | The key of the secret containing the CA certificate (in PEM format) used for SSL authentication. |
 | kafka.external | bool | `false` | If false, deploy a Kafka cluster together with the platform services. Otherwise, use an existing Kafka cluster. |
+| kafka.kraft | string | `"enabled"` | It can be: enabled, disabled, migration, rollback. See https://strimzi.io/docs/operators/latest/deploying#proc-deploy-migrate-kraft-str |
 | kafka.krb5Config | string | `""` | [external] Used only when saslMechanism is set to "GSSAPI". The Keberos configuration file used for SASL GSSAPI authentication. |
 | kafka.nameOverride | string | `""` | with the provided value. |
 | kafka.rack.topologyKey | string | `"topology.kubernetes.io/zone"` |  |

nebuly-platform/templates/_kafka.tpl

@@ -6,6 +6,10 @@
 {{ .Values.kafka.user }}
 {{- end -}}
 
+{{- define "kafka.kraft" -}}
+{{ .Values.kafka.kraft | default "enabled" }}
+{{- end -}}
+
 {{- define "kafka.bootstrapServers" -}}
 {{- if .Values.kafka.external -}}
 {{ .Values.kafka.bootstrapServers }}

nebuly-platform/templates/kafka-cluster_controller_nodepool.yaml

@@ -0,0 +1,30 @@
+{{ if not .Values.kafka.external -}}
+apiVersion: kafka.strimzi.io/v1beta2
+kind: KafkaNodePool
+metadata:
+  name: controller
+  namespace: {{ include "nebuly-platform.namespace" . }}
+  labels:
+    strimzi.io/cluster: {{ include "kafka.fullname" . }}
+    {{- include "nebuly-platform.selectorLabels" . | nindent 4 }}
+  annotations:
+    {{- include "nebuly-platform.annotations" . | nindent 4 }}
+spec:
+  replicas: {{ .Values.kafka.controller.replicas }}
+  roles:
+    - controller
+  storage:
+    type: persistent-claim
+    size: {{ .Values.kafka.controller.storage.size }}
+    {{- with .Values.kafka.controller.storage.deleteClaim }}
+    deleteClaim: {{ . }}
+    {{- end }}
+    {{- with .Values.kafka.controller.storage.class }}
+    class: {{ . }}
+    {{- end }}
+  resources:
+    {{- toYaml .Values.kafka.controller.resources | nindent 4 }}
+  template:
+    pod:
+      affinity: {{- toYaml .Values.kafka.controller.affinity | nindent 8 }}
+{{- end }}

nebuly-platform/templates/kafka-cluster_main.yaml

@@ -6,6 +6,7 @@ metadata:
   namespace: {{ include "nebuly-platform.namespace" . }}
   annotations:
     strimzi.io/node-pools: enabled
+    strimzi.io/kraft: {{ include "kafka.kraft" . }}
     {{- include "nebuly-platform.annotations" . | nindent 4 }}
 spec:
   zookeeper:

nebuly-platform/values.yaml

@@ -833,6 +833,10 @@ kafka:
   # -- The number of Kafka brokers in the cluster.
   replicas: 3
 
+  # -- KRaft mode, if true, deploy Kafka in KRaft mode (without Zookeeper).
+  # -- It can be: enabled, disabled, migration, rollback. See https://strimzi.io/docs/operators/latest/deploying#proc-deploy-migrate-kraft-str
+  kraft: enabled
+
   affinity: { }
 
   # -- The storage class used for the Kafka and Zookeeper storage.
@@ -856,6 +860,23 @@ kafka:
     requests:
       cpu: 100m
       memory: 1024Mi
+  # Settings of the Controller NodePool.
+  # @default -- -
+  controller:
+    replicas: 3
+    affinity: { }
+    storage:
+      type: persistent-claim
+      size: 10Gi
+      deleteClaim: false
+    # Resources of the Controller pods.
+    # @default -- -
+    resources:
+      limits:
+        memory: 2048Mi
+      requests:
+        cpu: 100m
+        memory: 1024Mi
   # Settings of the Zookeeper cluster.
   # @default -- -
   zookeeper: