kafka: switch to apache/kafka container (#28077)

mshima · web-flow · commit 03d45f9482df · 2024-12-04T22:39:33.000Z
diff --git a/generators/app/__snapshots__/generator.spec.ts.snap b/generators/app/__snapshots__/generator.spec.ts.snap
@@ -329,12 +329,11 @@ exports[`generator - app with default config should match snapshot 1`] = `
   "devServerPort": 4200,
   "devServerPortProxy": 9000,
   "dockerContainers": {
+    "apache/kafka-native": "apache-kafka-native-placeholder",
     "apachepulsar/pulsar": "apachepulsar-pulsar-placeholder",
     "cassandra": "cassandra-placeholder",
     "cassandraImage": "cassandra-image-placeholder",
     "cassandraTag": "cassandra-tag-placeholder",
-    "confluentinc/cp-kafka": "confluentinc-cp-kafka-placeholder",
-    "confluentinc/cp-zookeeper": "confluentinc-cp-zookeeper-placeholder",
     "consul": "consul-placeholder",
     "consulConfigLoader": "consul-config-loader-placeholder",
     "consulConfigLoaderImage": "consul-config-loader-image-placeholder",
@@ -449,9 +448,6 @@ exports[`generator - app with default config should match snapshot 1`] = `
     "zipkin": "zipkin-placeholder",
     "zipkinImage": "zipkin-image-placeholder",
     "zipkinTag": "zipkin-tag-placeholder",
-    "zookeeper": "zookeeper-placeholder",
-    "zookeeperImage": "zookeeper-image-placeholder",
-    "zookeeperTag": "zookeeper-tag-placeholder",
   },
   "dockerServices": [
     "app",
@@ -999,12 +995,11 @@ exports[`generator - app with gateway should match snapshot 1`] = `
   "devServerPort": 4200,
   "devServerPortProxy": 9000,
   "dockerContainers": {
+    "apache/kafka-native": "apache-kafka-native-placeholder",
     "apachepulsar/pulsar": "apachepulsar-pulsar-placeholder",
     "cassandra": "cassandra-placeholder",
     "cassandraImage": "cassandra-image-placeholder",
     "cassandraTag": "cassandra-tag-placeholder",
-    "confluentinc/cp-kafka": "confluentinc-cp-kafka-placeholder",
-    "confluentinc/cp-zookeeper": "confluentinc-cp-zookeeper-placeholder",
     "consul": "consul-placeholder",
     "consulConfigLoader": "consul-config-loader-placeholder",
     "consulConfigLoaderImage": "consul-config-loader-image-placeholder",
@@ -1119,9 +1114,6 @@ exports[`generator - app with gateway should match snapshot 1`] = `
     "zipkin": "zipkin-placeholder",
     "zipkinImage": "zipkin-image-placeholder",
     "zipkinTag": "zipkin-tag-placeholder",
-    "zookeeper": "zookeeper-placeholder",
-    "zookeeperImage": "zookeeper-image-placeholder",
-    "zookeeperTag": "zookeeper-tag-placeholder",
   },
   "dockerServices": [
     "app",
@@ -1664,12 +1656,11 @@ exports[`generator - app with microservice should match snapshot 1`] = `
   "devServerPort": undefined,
   "devServerPortProxy": undefined,
   "dockerContainers": {
+    "apache/kafka-native": "apache-kafka-native-placeholder",
     "apachepulsar/pulsar": "apachepulsar-pulsar-placeholder",
     "cassandra": "cassandra-placeholder",
     "cassandraImage": "cassandra-image-placeholder",
     "cassandraTag": "cassandra-tag-placeholder",
-    "confluentinc/cp-kafka": "confluentinc-cp-kafka-placeholder",
-    "confluentinc/cp-zookeeper": "confluentinc-cp-zookeeper-placeholder",
     "consul": "consul-placeholder",
     "consulConfigLoader": "consul-config-loader-placeholder",
     "consulConfigLoaderImage": "consul-config-loader-image-placeholder",
@@ -1784,9 +1775,6 @@ exports[`generator - app with microservice should match snapshot 1`] = `
     "zipkin": "zipkin-placeholder",
     "zipkinImage": "zipkin-image-placeholder",
     "zipkinTag": "zipkin-tag-placeholder",
-    "zookeeper": "zookeeper-placeholder",
-    "zookeeperImage": "zookeeper-image-placeholder",
-    "zookeeperTag": "zookeeper-tag-placeholder",
   },
   "dockerServices": [
     "app",
diff --git a/generators/docker-compose/templates/README-DOCKER-COMPOSE.md.ejs b/generators/docker-compose/templates/README-DOCKER-COMPOSE.md.ejs
@@ -32,7 +32,6 @@ Launch all your infrastructure by running: `docker compose up -d`.
 <%_ } _%>
 <%_ if (useKafka === true) { _%>
 - Kafka
-- Zookeeper
 <%_ } _%>
 <%_ if (usePulsar === true) { _%>
 - Pulsar
diff --git a/generators/docker-compose/templates/docker-compose.yml.ejs b/generators/docker-compose/templates/docker-compose.yml.ejs
@@ -21,19 +21,17 @@ services:
 <%- appsYaml[i] %>
 <%_ } _%>
 <%_ if (useKafka) { _%>
-  zookeeper:
-    image: <%- dockerContainers.zookeeper %>
-    environment:
-      ZOOKEEPER_CLIENT_PORT: 2181
-      ZOOKEEPER_TICK_TIME: 2000
-      ZOOKEEPER_SYNC_LIMIT: 2
   kafka:
     image: <%- dockerContainers.kafka %>
     environment:
+      KAFKA_NODE_ID: 1
+      KAFKA_PROCESS_ROLES: broker,controller
       KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092
-      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
-      KAFKA_BROKER_ID: 2
-      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+      KAFKA_LISTENERS: PLAINTEXT://:9092,CONTROLLER://localhost:9093
+      KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT
+      KAFKA_CONTROLLER_QUORUM_VOTERS: 1@localhost:9093
+      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
       #ports:
       # - 9092:9092 # Uncomment to make Kafka available externally
 <%_ } _%>
diff --git a/generators/docker/__snapshots__/generator.spec.ts.snap b/generators/docker/__snapshots__/generator.spec.ts.snap
@@ -256,10 +256,6 @@ services:
     extends:
       file: ./kafka.yml
       service: kafka
-  zookeeper:
-    extends:
-      file: ./kafka.yml
-      service: zookeeper
 ",
     "stateCleared": "modified",
   },
@@ -365,10 +361,6 @@ services:
     extends:
       file: ./kafka.yml
       service: kafka
-  zookeeper:
-    extends:
-      file: ./kafka.yml
-      service: zookeeper
 ",
     "stateCleared": "modified",
   },
@@ -743,10 +735,6 @@ services:
     extends:
       file: ./kafka.yml
       service: kafka
-  zookeeper:
-    extends:
-      file: ./kafka.yml
-      service: zookeeper
 ",
     "stateCleared": "modified",
   },
@@ -856,10 +844,6 @@ services:
     extends:
       file: ./kafka.yml
       service: kafka
-  zookeeper:
-    extends:
-      file: ./kafka.yml
-      service: zookeeper
 ",
     "stateCleared": "modified",
   },
diff --git a/generators/docker/generator.ts b/generators/docker/generator.ts
@@ -228,10 +228,7 @@ export default class DockerGenerator extends BaseApplicationGenerator {
           );
         }
         if (application.dockerServices.includes('kafka')) {
-          source.addDockerExtendedServiceToApplicationAndServices(
-            { serviceName: 'kafka' },
-            { serviceFile: './kafka.yml', serviceName: 'zookeeper' },
-          );
+          source.addDockerExtendedServiceToApplicationAndServices({ serviceName: 'kafka' });
         }
       },
 
diff --git a/generators/docker/templates/docker/kafka.yml.ejs b/generators/docker/templates/docker/kafka.yml.ejs
@@ -26,22 +26,14 @@ services:
     ports:
       - 127.0.0.1:9092:9092
     environment:
-      KAFKA_BROKER_ID: 1
-      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
-      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:9092
-      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
-      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
-      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
-      KAFKA_ADVERTISED_HOST_NAME: kafka
-<%_ if (backendTypeSpringBoot) { _%>
-    labels:
-      org.springframework.boot.ignore: true
-<%_ } _%>
-  zookeeper:
-    image: <%- dockerContainers.zookeeper %>
-    environment:
-      ZOOKEEPER_CLIENT_PORT: 2181
-      ZOOKEEPER_TICK_TIME: 2000
+      KAFKA_NODE_ID: 1
+      KAFKA_PROCESS_ROLES: broker,controller
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092
+      KAFKA_LISTENERS: PLAINTEXT://:9092,CONTROLLER://localhost:9093
+      KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT
+      KAFKA_CONTROLLER_QUORUM_VOTERS: 1@localhost:9093
+      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
 <%_ if (backendTypeSpringBoot) { _%>
     labels:
       org.springframework.boot.ignore: true
diff --git a/generators/kubernetes-helm/__snapshots__/kubernetes.helm.spec.ts.snap b/generators/kubernetes-helm/__snapshots__/kubernetes.helm.spec.ts.snap
@@ -116,7 +116,7 @@ dependencies:
   "csvc-helm/templates/_helpers.tpl": {
     "contents": "{{/* vim: set filetype=mustache: */}}
 {{/*
-Kafka and zookeeper customisation
+Kafka customisation
 */}}
 {{- define "kafka.name" -}}
 {{- default "jhipster-kafka" -}}
@@ -126,17 +126,6 @@ Kafka and zookeeper customisation
 {{- default "jhipster-kafka" -}}
 {{- end -}}
 
-{{- define "zookeeper.name" -}}
-{{- default "jhipster-zookeeper" -}}
-{{- end -}}
-
-{{- define "zookeeper.fullname" -}}
-{{- default "jhipster-zookeeper" -}}
-{{- end -}}
-
-{{- define "kafka.zookeeper.fullname" -}}
-{{- default "jhipster-zookeeper" -}}
-{{- end -}}
 
 ",
     "stateCleared": "modified",
@@ -145,9 +134,6 @@ Kafka and zookeeper customisation
     "contents": "kafka:
   enabled: true
   replicas: 3
-  zookeeper:
-    enabled: true
-    replicas: 3
 ",
     "stateCleared": "modified",
   },
diff --git a/generators/kubernetes-helm/templates/csvc/helpers.tpl.ejs b/generators/kubernetes-helm/templates/csvc/helpers.tpl.ejs
@@ -1,7 +1,7 @@
 {{/* vim: set filetype=mustache: */}}
 <%_ if (useKafka) { _%>
 {{/*
-Kafka and zookeeper customisation
+Kafka customisation
 */}}
 {{- define "kafka.name" -}}
 {{- default "jhipster-kafka" -}}
@@ -11,17 +11,6 @@ Kafka and zookeeper customisation
 {{- default "jhipster-kafka" -}}
 {{- end -}}
 
-{{- define "zookeeper.name" -}}
-{{- default "jhipster-zookeeper" -}}
-{{- end -}}
-
-{{- define "zookeeper.fullname" -}}
-{{- default "jhipster-zookeeper" -}}
-{{- end -}}
-
-{{- define "kafka.zookeeper.fullname" -}}
-{{- default "jhipster-zookeeper" -}}
-{{- end -}}
 <%_ } _%>
 <%_ if (monitoringPrometheus) { _%>
 {{/*
diff --git a/generators/kubernetes-helm/templates/csvc/values.yml.ejs b/generators/kubernetes-helm/templates/csvc/values.yml.ejs
@@ -6,13 +6,6 @@ kafka:
   podAnnotations:
     sidecar.istio.io/inject: "false"
   <%_ } _%>
-  zookeeper:
-    enabled: true
-    replicas: 3
-  <%_ if (istio) { _%>
-    podAnnotations:
-      sidecar.istio.io/inject: "false"
-  <%_ } _%>
 <%_ } _%>
 <%_ if (monitoringPrometheus) { _%>
 prometheus:
diff --git a/generators/kubernetes/__snapshots__/kubernetes.spec.ts.snap b/generators/kubernetes/__snapshots__/kubernetes.spec.ts.snap
@@ -203,14 +203,24 @@ spec:
       - name: kafka
         image: kafka-placeholder
         env:
+        - name: KAFKA_NODE_ID
+          value: 1
+        - name: KAFKA_PROCESS_ROLES
+          value: broker,controller
         - name: KAFKA_ADVERTISED_LISTENERS
           value: 'PLAINTEXT://jhipster-kafka.default.svc.cluster.local:9092'
-        - name: KAFKA_ZOOKEEPER_CONNECT
-          value: 'jhipster-zookeeper.default.svc.cluster.local:2181'
-        - name: KAFKA_BROKER_ID
-          value: '2'
+        - name: KAFKA_LISTENERS
+          value: PLAINTEXT://:9092,CONTROLLER://localhost:9093
+        - name: KAFKA_CONTROLLER_LISTENER_NAMES
+          value: CONTROLLER
+        - name: KAFKA_LISTENER_SECURITY_PROTOCOL_MAP
+          value: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT
+        - name: KAFKA_CONTROLLER_QUORUM_VOTERS
+          value: 1@localhost:9093
+        - name: KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS
+          value: 0
         - name: KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR
-          value: '1'
+          value: 1
         ports:
         - containerPort: 9092
         resources:
@@ -231,53 +241,7 @@ spec:
     app: jhipster-kafka
   ports:
   - port: 9092
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: jhipster-zookeeper
-  namespace: default
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: jhipster-zookeeper
-  template:
-    metadata:
-      labels:
-        app: jhipster-zookeeper
-    spec:
-      containers:
-      - name: zookeeper
-        image: zookeeper-placeholder
-        env:
-        - name: ZOOKEEPER_CLIENT_PORT
-          value: '2181'
-        - name: ZOOKEEPER_TICK_TIME
-          value: '2000'
-        - name: ZOOKEEPER_SYNC_LIMIT
-          value: '2'
-        ports:
-        - containerPort: 2181
-        resources:
-          requests:
-            memory: "512Mi"
-            cpu: "500m"
-          limits:
-            memory: "1Gi"
-            cpu: "1"
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: jhipster-zookeeper
-  namespace: default
-spec:
-  selector:
-    app: jhipster-zookeeper
-  ports:
-  - port: 2181
-",
+  ",
     "stateCleared": "modified",
   },
   "samplekafka-k8s/samplekafka-deployment.yml": {
diff --git a/generators/kubernetes/templates/messagebroker/kafka.yml.ejs b/generators/kubernetes/templates/messagebroker/kafka.yml.ejs
diff --git a/generators/server/resources/Dockerfile b/generators/server/resources/Dockerfile
diff --git a/generators/spring-cloud-stream/generators/kafka/templates/src/test/java/_package_/config/KafkaTestContainer.java.ejs b/generators/spring-cloud-stream/generators/kafka/templates/src/test/java/_package_/config/KafkaTestContainer.java.ejs
diff --git a/generators/spring-cloud-stream/generators/kafka/templates/src/test/java/_package_/config/KafkaTestContainersSpringContextCustomizerFactory.java.ejs b/generators/spring-cloud-stream/generators/kafka/templates/src/test/java/_package_/config/KafkaTestContainersSpringContextCustomizerFactory.java.ejs

Original file line number	Diff line number	Diff line change
`@@ -228,10 +228,7 @@ export default class DockerGenerator extends BaseApplicationGenerator {`
`228`	`228`	`);`
`229`	`229`	`}`
`230`	`230`	`if (application.dockerServices.includes('kafka')) {`
`231`		`- source.addDockerExtendedServiceToApplicationAndServices(`
`232`		`- { serviceName: 'kafka' },`
`233`		`- { serviceFile: './kafka.yml', serviceName: 'zookeeper' },`
`234`		`- );`
	`231`	`+ source.addDockerExtendedServiceToApplicationAndServices({ serviceName: 'kafka' });`
`235`	`232`	`}`
`236`	`233`	`},`
`237`	`234`