camunda-consulting · falko · Jan 9, 2026 · Jan 9, 2026
diff --git a/src/main/resources/benchmarktemplates/Makefile b/src/main/resources/benchmarktemplates/Makefile
@@ -18,14 +18,22 @@ payload ?= ${loadGeneratorStarter.payload}
 scenario ?= msg-scenario.json
 #scenario ?= msg-scenario-references.json
 
+engineImage ?= ${engine.image}
+gatewayReplicas ?= ${gateway.replicas}
+gatewayPinning ?= ${gateway.pinning}
+
 .PHONY: all
-all: chart sync-image install-camunda chart-infos cpu-info.txt pods await-zeebe deploy-models rebalance-leaders ${chaosTarget} benchmark await-benchmark
+all: chart sync-image install-camunda chart-infos cpu-info.txt pods await-zeebe await-gateway deploy-models rebalance-leaders ${chaosTarget} benchmark await-benchmark
 #all: chart sync-image install-camunda chart-infos cpu-info.txt pods await-zeebe deploy-models rebalance-leaders ${chaosTarget} await-elasticsearch await-webapps benchmark await-benchmark
 
 .PHONY: clean
 clean: clean-benchmark clean-job-deploy-models uninstall-camunda
 
-ifeq ($(OS),Windows_NT)
+# Set the timeout command based on the OS
+# On MacOS `brew install coreutils` provides `gtimeout`
+TIMEOUT_CMD := $(shell uname | grep -q 'Darwin' && echo 'gtimeout' || echo 'timeout')
+
+ifeq ("$(OS)", "Windows_NT")
     root ?= $(CURDIR)/../../../../camunda-8-helm-profiles
 else
     root ?= $(shell pwd)/../../../../camunda-8-helm-profiles
@@ -36,9 +44,9 @@ include $(root)/metrics/metrics.mk # for Grafana URL
 
 .PHONY: sync-image
 sync-image:
-  ifneq '${engine.image}' 'camunda/zeebe'
+ifneq ("$(engineImage)", "camunda/zeebe")
 	docker manifest inspect gcr.io/camunda-researchanddevelopment/falko-zeebe:${engine.engineVersion} || docker buildx imagetools create --tag gcr.io/camunda-researchanddevelopment/falko-zeebe:${engine.engineVersion} gcr.io/zeebe-io/zeebe:${engine.engineVersion}
-  endif
+endif
 
 #: Deploy BPMN and DMN models using zbctl in a k8s job
 deploy-models: copy-models job-deploy-models logs-job-deploy-models clean-job-deploy-models
@@ -79,26 +87,99 @@ deploy-chaos-gateway:
 deploy-chaos-broker:
 	kubectl apply -f chaos-network-brokers.yaml -n $(namespace)	
 
+.PHONY: generate-benchmark-configs # Generate benchmark configs with dynamic gateway IPs
+generate-benchmark-configs:
+ifeq ("$(gatewayPinning)", "TRUE")
+	@echo "Generating benchmark configs with dynamic gateway IPs..."
+	@GATEWAY_IPS=$$(kubectl get pods -l app.kubernetes.io/component=zeebe-gateway -n $(namespace) -o jsonpath='{.items[*].status.podIP}' | tr ' ' '\n' | sort); \
+	IP_COUNT=$$(echo "$$GATEWAY_IPS" | wc -l); \
+	if [ $$IP_COUNT -eq 0 ]; then \
+		echo "No gateway IPs found!"; \
+		exit 1; \
+	fi; \
+	i=0; \
+	while [ $$i -lt $(gatewayReplicas) ]; do \
+		IP_INDEX=$$((i % IP_COUNT)); \
+		GATEWAY_IP=$$(echo "$$GATEWAY_IPS" | sed -n "$$((IP_INDEX + 1))p"); \
+		export GATEWAY_IP=$$GATEWAY_IP; \
+		export BENCHMARK_ID=$$i; \
+		envsubst < benchmark-gateway-template.yaml > benchmark-gateway-$$i.yaml; \
+		echo "Generated benchmark-gateway-$$i.yaml with IP $$GATEWAY_IP (round-robin $$IP_INDEX)"; \
+		i=$$((i + 1)); \
+	done
+endif
+
 .PHONY: benchmark
-benchmark: namespace copy-models
-	kubectl create configmap payload      --from-file=$(payload)=models/$(payload)       -n $(namespace)
+benchmark: namespace copy-models generate-benchmark-configs
+ifeq ("$(gatewayPinning)", "TRUE")
+	kubectl apply -f camunda-zeebe-gateway-headless.yaml                                   -n $(namespace)
+endif
+	kubectl create configmap payload      --from-file=$(payload)=models/$(payload)         -n $(namespace)
 #	kubectl create configmap msg-scenario --from-file=msg-scenario.json=models/msg-scenario.json -n $(namespace)
 	kubectl create configmap msg-scenario --from-file=msg-scenario.json=models/$(scenario) -n $(namespace)
+ifeq ("$(gatewayPinning)", "TRUE")
+	i=0; \
+	while [ $$i -lt $(gatewayReplicas) ]; do \
+		kubectl apply -f benchmark-gateway-$$i.yaml                                        -n $(namespace); \
+		i=$$((i + 1)); \
+	done
+else
 	kubectl apply -f $(benchmark)                                                          -n $(namespace)
-	kubectl expose deployment benchmark --type=ClusterIP   --name=benchmark-svc  --port=8088 --target-port=8088
-	kubectl apply -f benchmark-service-monitor.yaml -n $(namespace)
+#	kubectl expose deployment benchmark --type=ClusterIP   --name=benchmark-svc  --port=8088 --target-port=8088 -n $(namespace)
+	kubectl apply -f benchmark-service-monitor.yaml                                        -n $(namespace)
+endif
 # TODO automatically switch/generate benchmark config
 
 .PHONY: clean-benchmark
 clean-benchmark:
-	-kubectl delete -f $(benchmark)        -n $(namespace)
-	-kubectl delete configmap payload      -n $(namespace)
-	-kubectl delete configmap msg-scenario -n $(namespace)
+ifeq ("$(gatewayPinning)", "TRUE")
+	i=0; \
+	while [ $$i -lt $(gatewayReplicas) ]; do \
+		kubectl delete -f benchmark-gateway-$$i.yaml -n $(namespace); \
+		i=$$((i + 1)); \
+	done
+else
+	-kubectl delete service benchmark-svc             -n $(namespace)
+	-kubectl delete -f $(benchmark)                   -n $(namespace)
+endif
+	-kubectl delete configmap payload                 -n $(namespace)
+	-kubectl delete configmap msg-scenario            -n $(namespace)
 
 .PHONY: await-benchmark
 await-benchmark:
-	kubectl wait --for=condition=Ready pod -n $(namespace) -l app=benchmark --timeout=900s
+ifeq ("$(gatewayPinning)", "TRUE")
+	i=0; \
+	while [ $$i -lt $(gatewayReplicas) ]; do \
+		kubectl rollout status --watch statefulset/benchmark-gateway-$$i --timeout=900s -n $(namespace); \
+		i=$$((i + 1)); \
+	done
+else
+#	kubectl rollout status --watch deployment/benchmark --timeout=900s -n $(namespace)
+	kubectl rollout status --watch statefulset/benchmark --timeout=900s -n $(namespace)
+endif
 
 .PHONY: logs-benchmark
 logs-benchmark:
-	kubectl logs -f -l app=benchmark -n $(namespace)
+ifeq ("$(gatewayPinning)", "TRUE")
+	selector=""; i=0; \
+	while [ $$i -lt $(gatewayReplicas) ]; do \
+	  if [ -z "$$selector" ]; then selector="benchmark-gateway-$$i"; else selector="$$selector,benchmark-gateway-$$i"; fi; \
+	  i=$$((i+1)); \
+	done; \
+	kubectl logs -f -l "app in ($$selector)" -n $(namespace)
+else
+	kubectl logs -f -l app=benchmark -n $(namespace)
+endif
+
+.PHONY: logs-benchmark-timed
+logs-benchmark-timed:
+ifeq ("$(gatewayPinning)", "TRUE")
+#	stern benchmark-gateway-[0-$(gatewayReplicas)] -n $(namespace)
+	@$(TIMEOUT_CMD) ${loadGeneratorStarter.runDuration}s stern benchmark-gateway-[0-$(gatewayReplicas)] -n $(namespace) \
+	|| { exit_code=$$?; [ $$exit_code -eq 124 ] && echo "${loadGeneratorStarter.runDuration}s timeout reached. Disconnecting from the bechmark logs." || exit $$exit_code; }
+else
+#	@$(TIMEOUT_CMD) 1200s kubectl logs -f -l "app in (benchmark-gateway-1,benchmark-gateway-2,benchmark-gateway-3,benchmark-gateway-4)" --max-log-requests=8 -n $(namespace) \
+	|| { exit_code=$$?; [ $$exit_code -eq 124 ] && echo "1200s timeout reached. Disconnecting from the bechmark logs." || exit $$exit_code; }
+	@$(TIMEOUT_CMD) ${loadGeneratorStarter.runDuration}s stern benchmark -n $(namespace) \
+	|| { exit_code=$$?; [ $$exit_code -eq 124 ] && echo "${loadGeneratorStarter.runDuration}s timeout reached. Disconnecting from the bechmark logs." || exit $$exit_code; }
+endif
diff --git a/src/main/resources/benchmarktemplates/benchmark-gateway-template.yaml b/src/main/resources/benchmarktemplates/benchmark-gateway-template.yaml
@@ -0,0 +1,74 @@
+# config for https://github.com/camunda-community-hub/camunda-8-benchmark
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: benchmark-gateway-${BENCHMARK_ID}
+  labels:
+    app: benchmark-gateway-${BENCHMARK_ID}
+spec:
+  selector:
+    matchLabels:
+      app: benchmark-gateway-${BENCHMARK_ID}
+  serviceName: benchmark-gateway-${BENCHMARK_ID}
+  replicas: 1
+  template:
+    metadata:
+#      annotations:
+#        linkerd.io/inject: enabled
+#        config.linkerd.io/opaque-ports: "26500"
+      labels:
+        app: benchmark-gateway-${BENCHMARK_ID}
+#        istio.io/dataplane-mode: ambient
+    spec:
+      containers:
+      - name: benchmark
+        image: camundacommunityhub/camunda-8-benchmark:main
+        imagePullPolicy: Always
+        env:
+          - name: BENCHMARK_STARTER_ID
+            value: benchmark-gateway-${BENCHMARK_ID}
+          - name: JAVA_OPTIONS
+            value: >-
+              -Dcamunda.client.mode=selfmanaged
+              -Dcamunda.client.zeebe.grpc-address=http://${GATEWAY_IP}:26500            
+              -Dcamunda.client.zeebe.prefer-rest-over-grpc=false
+              -Dcamunda.client.zeebe.request-timeout=600s
+              -Dcamunda.client.zeebe.defaults.poll-interval=1ms
+              -Dcamunda.client.zeebe.default-job-worker-stream-enabled=true
+              -Dcamunda.client.worker.defaults.stream-enabled=true
+              -Dbenchmark.startProcesses=true
+              -Dbenchmark.bpmnProcessId=${loadGeneratorStarter.processModel}
+              -Dbenchmark.startPiPerSecond=${loadGeneratorStarter.startThroughput}
+              -Dbenchmark.startPiIncreaseFactor=${loadGeneratorStarter.startPiIncreaseFactor}
+              -Dbenchmark.startRateAdjustmentStrategy=${loadGeneratorStarter.rateAdjustmentStrategy}
+              -Dbenchmark.fixedBackOffDelay=${jobWorker.fixedBackOffDelay}
+              -Dbenchmark.maxBackpressurePercentage=1.0
+              -Dbenchmark.startWorkers=true
+              -Dbenchmark.jobType=benchmark-task
+              -Dbenchmark.multipleJobTypes=${jobWorker.numberOfJobTypes}
+              -Dbenchmark.taskCompletionDelay=${jobWorker.jobDuration}
+              -Dbenchmark.payloadPath=file:/${loadGeneratorStarter.payload}
+              -Dbenchmark.autoDeployProcess=false
+              -Dbenchmark.warmupPhaseDurationMillis=${loadGeneratorStarter.rampUpTime}000
+              -Dbenchmark.enablePartitionPinning=${jobWorker.partitionPinning}
+              -Dbenchmark.partitionCount=${engine.partitions}
+              -Dbenchmark.numberOfStarters=${loadGeneratorStarter.starterReplicas}
+#              -Dcamunda.client.zeebe.grpc-address=http://camunda-zeebe-gateway-grpc:26500            
+#              -Dzeebe.client.worker.max-jobs-active=${jobWorker.maxJobsActive}
+#              -Dzeebe.client.worker.threads=${jobWorker.numberOfThreads}
+        resources:
+          limits:
+            cpu: ${engine.vcpus}
+            memory: ${engine.ram}Gi
+          requests:
+            cpu: 5
+            memory: 5Gi
+        volumeMounts:
+        - name: payload
+          mountPath: ${loadGeneratorStarter.payload}
+          subPath: ${loadGeneratorStarter.payload}
+          readOnly: true
+      volumes:
+      - name: payload
+        configMap:
+          name: payload
diff --git a/src/main/resources/benchmarktemplates/benchmark-msg-with-jobs.yaml b/src/main/resources/benchmarktemplates/benchmark-msg-with-jobs.yaml
@@ -22,24 +22,27 @@ spec:
         env:
           - name: JAVA_OPTIONS
             value: >-
-              -Dzeebe.client.broker.gateway-address=camunda-zeebe-gateway:26500
-              -Dzeebe.client.security.plaintext=true
-              -Dzeebe.client.request-timeout=600s
-              -Dzeebe.client.job.poll-interval=1ms
-              -Dzeebe.client.default-job-worker-stream-enabled=true
+              -Dcamunda.client.mode=selfmanaged
+              -Dcamunda.client.zeebe.grpc-address=http://camunda-zeebe-gateway:26500
+              -Dcamunda.client.zeebe.prefer-rest-over-grpc=false
+              -Dcamunda.client.zeebe.request-timeout=600s
+              -Dcamunda.client.zeebe.defaults.poll-interval=1ms
+              -Dcamunda.client.zeebe.default-job-worker-stream-enabled=true
+              -Dcamunda.client.worker.defaults.stream-enabled=true
               -Dbenchmark.startProcesses=false
               -Dbenchmark.startWorkers=true
               -Dbenchmark.jobType=benchmark-task
               -Dbenchmark.multipleJobTypes=${jobWorker.numberOfJobTypes}
-              -Dbenchmark.payloadPath=file:/payload.json
               -Dbenchmark.taskCompletionDelay=${jobWorker.jobDuration}
+              -Dbenchmark.payloadPath=file:/${loadGeneratorStarter.payload}
               -Dbenchmark.fixedBackOffDelay=${jobWorker.fixedBackOffDelay}
               -Dbenchmark.messageScenario=file:/msg-scenario.json
               -Dbenchmark.messagesTtl=${loadGeneratorStarter.messageTtl}
               -Dbenchmark.messagesScenariosPerSecond=${loadGeneratorStarter.startThroughput}
               -Dbenchmark.delayBetweenMessages=1
               -Dbenchmark.messagesLoadDuration=${loadGeneratorStarter.runDuration}000
               -Dbenchmark.autoDeployProcess=false
+              -Dbenchmark.warmupPhaseDurationMillis=${loadGeneratorStarter.rampUpTime}000
 # TODO make this work: -Dzeebe.client.worker.max-jobs-active=${jobWorker.maxJobsActive}
         resources:
           limits:

diff --git a/src/main/resources/benchmarktemplates/benchmark-msg.yaml b/src/main/resources/benchmarktemplates/benchmark-msg.yaml
@@ -22,8 +22,9 @@ spec:
         env:
           - name: JAVA_OPTIONS
             value: >-
-              -Dzeebe.client.broker.gateway-address=camunda-zeebe-gateway:26500
-              -Dzeebe.client.security.plaintext=true
+              -Dcamunda.client.mode=selfmanaged
+              -Dcamunda.client.zeebe.grpc-address=http://camunda-zeebe-gateway:26500
+              -Dcamunda.client.zeebe.prefer-rest-over-grpc=false
               -Dbenchmark.startProcesses=false
               -Dbenchmark.startWorkers=false
               -Dbenchmark.messageScenario=file:/msg-scenario.json
@@ -32,6 +33,7 @@ spec:
               -Dbenchmark.delayBetweenMessages=1
               -Dbenchmark.messagesLoadDuration=${loadGeneratorStarter.runDuration}000
               -Dbenchmark.autoDeployProcess=false
+              -Dbenchmark.warmupPhaseDurationMillis=${loadGeneratorStarter.rampUpTime}000
         resources:
           limits:
             cpu: ${engine.vcpus}

diff --git a/src/main/resources/benchmarktemplates/benchmark.yaml b/src/main/resources/benchmarktemplates/benchmark.yaml
@@ -1,6 +1,6 @@
 # config for https://github.com/camunda-community-hub/camunda-8-benchmark
 apiVersion: apps/v1
-kind: Deployment
+kind: StatefulSet
 metadata:
   name: benchmark
   labels:
@@ -9,47 +9,63 @@ spec:
   selector:
     matchLabels:
       app: benchmark
+  serviceName: benchmark
   replicas: ${loadGeneratorStarter.starterReplicas}
   template:
     metadata:
+#      annotations:
+#        linkerd.io/inject: enabled
+#        config.linkerd.io/opaque-ports: "26500"
       labels:
         app: benchmark
+#        istio.io/dataplane-mode: ambient
     spec:
       containers:
       - name: benchmark
         image: camundacommunityhub/camunda-8-benchmark:main
         imagePullPolicy: Always
         env:
+          - name: BENCHMARK_STARTER_ID
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.name
           - name: JAVA_OPTIONS
             value: >-
               -Dcamunda.client.mode=selfmanaged
-              -Dcamunda.client.zeebe.grpc-address=http://camunda-zeebe-gateway:26500            
+              -Dcamunda.client.zeebe.grpc-address=http://camunda-zeebe-gateway:26500
               -Dcamunda.client.zeebe.prefer-rest-over-grpc=false
-              -Dzeebe.client.request-timeout=600s
-              -Dzeebe.client.job.poll-interval=1ms
+              -Dcamunda.client.zeebe.request-timeout=600s
+              -Dcamunda.client.zeebe.defaults.poll-interval=1ms
               -Dcamunda.client.zeebe.default-job-worker-stream-enabled=true
+              -Dcamunda.client.worker.defaults.stream-enabled=true
               -Dbenchmark.startProcesses=true
               -Dbenchmark.bpmnProcessId=${loadGeneratorStarter.processModel}
               -Dbenchmark.startPiPerSecond=${loadGeneratorStarter.startThroughput}
               -Dbenchmark.startPiIncreaseFactor=${loadGeneratorStarter.startPiIncreaseFactor}
               -Dbenchmark.startRateAdjustmentStrategy=${loadGeneratorStarter.rateAdjustmentStrategy}
               -Dbenchmark.fixedBackOffDelay=${jobWorker.fixedBackOffDelay}
               -Dbenchmark.maxBackpressurePercentage=1.0
+              -Dbenchmark.startWorkers=true
               -Dbenchmark.jobType=benchmark-task
               -Dbenchmark.multipleJobTypes=${jobWorker.numberOfJobTypes}
               -Dbenchmark.taskCompletionDelay=${jobWorker.jobDuration}
               -Dbenchmark.payloadPath=file:/${loadGeneratorStarter.payload}
+              -Dbenchmark.fixedBackOffDelay=${jobWorker.fixedBackOffDelay}
               -Dbenchmark.autoDeployProcess=false
               -Dbenchmark.warmupPhaseDurationMillis=${loadGeneratorStarter.rampUpTime}000
+              -Dbenchmark.enablePartitionPinning=${jobWorker.partitionPinning}
+              -Dbenchmark.partitionCount=${engine.partitions}
+              -Dbenchmark.numberOfStarters=${loadGeneratorStarter.starterReplicas}
+#              -Dcamunda.client.zeebe.grpc-address=http://camunda-zeebe-gateway-grpc:26500            
 #              -Dzeebe.client.worker.max-jobs-active=${jobWorker.maxJobsActive}
 #              -Dzeebe.client.worker.threads=${jobWorker.numberOfThreads}
         resources:
           limits:
             cpu: ${engine.vcpus}
             memory: ${engine.ram}Gi
           requests:
-            cpu: 1
-            memory: 1Gi
+            cpu: 100m
+            memory: 512Mi
         volumeMounts:
         - name: payload
           mountPath: ${loadGeneratorStarter.payload}