From c7fa0bb44a43bccc32585a4f4a6f38eacab6d50e Mon Sep 17 00:00:00 2001 From: Falko Menge Date: Fri, 9 Jan 2026 09:12:54 +0100 Subject: [PATCH 1/2] Update benchmark properties --- .../benchmark-msg-with-jobs.yaml | 15 +++++++++------ .../benchmarktemplates/benchmark-msg.yaml | 6 ++++-- .../resources/benchmarktemplates/benchmark.yaml | 6 ++++-- 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/src/main/resources/benchmarktemplates/benchmark-msg-with-jobs.yaml b/src/main/resources/benchmarktemplates/benchmark-msg-with-jobs.yaml index 05e6dd4..a5c1818 100644 --- a/src/main/resources/benchmarktemplates/benchmark-msg-with-jobs.yaml +++ b/src/main/resources/benchmarktemplates/benchmark-msg-with-jobs.yaml @@ -22,17 +22,19 @@ spec: env: - name: JAVA_OPTIONS value: >- - -Dzeebe.client.broker.gateway-address=camunda-zeebe-gateway:26500 - -Dzeebe.client.security.plaintext=true - -Dzeebe.client.request-timeout=600s - -Dzeebe.client.job.poll-interval=1ms - -Dzeebe.client.default-job-worker-stream-enabled=true + -Dcamunda.client.mode=selfmanaged + -Dcamunda.client.zeebe.grpc-address=http://camunda-zeebe-gateway:26500 + -Dcamunda.client.zeebe.prefer-rest-over-grpc=false + -Dcamunda.client.zeebe.request-timeout=600s + -Dcamunda.client.zeebe.defaults.poll-interval=1ms + -Dcamunda.client.zeebe.default-job-worker-stream-enabled=true + -Dcamunda.client.worker.defaults.stream-enabled=true -Dbenchmark.startProcesses=false -Dbenchmark.startWorkers=true -Dbenchmark.jobType=benchmark-task -Dbenchmark.multipleJobTypes=${jobWorker.numberOfJobTypes} - -Dbenchmark.payloadPath=file:/payload.json -Dbenchmark.taskCompletionDelay=${jobWorker.jobDuration} + -Dbenchmark.payloadPath=file:/${loadGeneratorStarter.payload} -Dbenchmark.fixedBackOffDelay=${jobWorker.fixedBackOffDelay} -Dbenchmark.messageScenario=file:/msg-scenario.json -Dbenchmark.messagesTtl=${loadGeneratorStarter.messageTtl} @@ -40,6 +42,7 @@ spec: -Dbenchmark.delayBetweenMessages=1 -Dbenchmark.messagesLoadDuration=${loadGeneratorStarter.runDuration}000 -Dbenchmark.autoDeployProcess=false + -Dbenchmark.warmupPhaseDurationMillis=${loadGeneratorStarter.rampUpTime}000 # TODO make this work: -Dzeebe.client.worker.max-jobs-active=${jobWorker.maxJobsActive} resources: limits: diff --git a/src/main/resources/benchmarktemplates/benchmark-msg.yaml b/src/main/resources/benchmarktemplates/benchmark-msg.yaml index 9295b23..1c738ae 100644 --- a/src/main/resources/benchmarktemplates/benchmark-msg.yaml +++ b/src/main/resources/benchmarktemplates/benchmark-msg.yaml @@ -22,8 +22,9 @@ spec: env: - name: JAVA_OPTIONS value: >- - -Dzeebe.client.broker.gateway-address=camunda-zeebe-gateway:26500 - -Dzeebe.client.security.plaintext=true + -Dcamunda.client.mode=selfmanaged + -Dcamunda.client.zeebe.grpc-address=http://camunda-zeebe-gateway:26500 + -Dcamunda.client.zeebe.prefer-rest-over-grpc=false -Dbenchmark.startProcesses=false -Dbenchmark.startWorkers=false -Dbenchmark.messageScenario=file:/msg-scenario.json @@ -32,6 +33,7 @@ spec: -Dbenchmark.delayBetweenMessages=1 -Dbenchmark.messagesLoadDuration=${loadGeneratorStarter.runDuration}000 -Dbenchmark.autoDeployProcess=false + -Dbenchmark.warmupPhaseDurationMillis=${loadGeneratorStarter.rampUpTime}000 resources: limits: cpu: ${engine.vcpus} diff --git a/src/main/resources/benchmarktemplates/benchmark.yaml b/src/main/resources/benchmarktemplates/benchmark.yaml index c66627b..0a3ae24 100644 --- a/src/main/resources/benchmarktemplates/benchmark.yaml +++ b/src/main/resources/benchmarktemplates/benchmark.yaml @@ -25,9 +25,10 @@ spec: -Dcamunda.client.mode=selfmanaged -Dcamunda.client.zeebe.grpc-address=http://camunda-zeebe-gateway:26500 -Dcamunda.client.zeebe.prefer-rest-over-grpc=false - -Dzeebe.client.request-timeout=600s - -Dzeebe.client.job.poll-interval=1ms + -Dcamunda.client.zeebe.request-timeout=600s + -Dcamunda.client.zeebe.defaults.poll-interval=1ms -Dcamunda.client.zeebe.default-job-worker-stream-enabled=true + -Dcamunda.client.worker.defaults.stream-enabled=true -Dbenchmark.startProcesses=true -Dbenchmark.bpmnProcessId=${loadGeneratorStarter.processModel} -Dbenchmark.startPiPerSecond=${loadGeneratorStarter.startThroughput} @@ -35,6 +36,7 @@ spec: -Dbenchmark.startRateAdjustmentStrategy=${loadGeneratorStarter.rateAdjustmentStrategy} -Dbenchmark.fixedBackOffDelay=${jobWorker.fixedBackOffDelay} -Dbenchmark.maxBackpressurePercentage=1.0 + -Dbenchmark.startWorkers=true -Dbenchmark.jobType=benchmark-task -Dbenchmark.multipleJobTypes=${jobWorker.numberOfJobTypes} -Dbenchmark.taskCompletionDelay=${jobWorker.jobDuration} From 8f24b8b2a3fa272b543e3afe07f04d29af2662f0 Mon Sep 17 00:00:00 2001 From: Falko Menge Date: Fri, 9 Jan 2026 09:19:54 +0100 Subject: [PATCH 2/2] Add Gateway & Partition Pinning --- .../resources/benchmarktemplates/Makefile | 107 ++++++++-- .../benchmark-gateway-template.yaml | 74 +++++++ .../benchmarktemplates/benchmark.yaml | 22 +- .../benchmarktemplates/camunda-values.yaml | 194 ++++++++++++++++-- .../camunda-zeebe-gateway-headless.yaml | 25 +++ .../benchmarktemplates/job-cpu-info.yaml | 4 +- src/main/resources/benchmarktemplates/run.sh | 8 +- 7 files changed, 397 insertions(+), 37 deletions(-) create mode 100644 src/main/resources/benchmarktemplates/benchmark-gateway-template.yaml create mode 100644 src/main/resources/benchmarktemplates/camunda-zeebe-gateway-headless.yaml diff --git a/src/main/resources/benchmarktemplates/Makefile b/src/main/resources/benchmarktemplates/Makefile index d31df81..15b6858 100644 --- a/src/main/resources/benchmarktemplates/Makefile +++ b/src/main/resources/benchmarktemplates/Makefile @@ -18,14 +18,22 @@ payload ?= ${loadGeneratorStarter.payload} scenario ?= msg-scenario.json #scenario ?= msg-scenario-references.json +engineImage ?= ${engine.image} +gatewayReplicas ?= ${gateway.replicas} +gatewayPinning ?= ${gateway.pinning} + .PHONY: all -all: chart sync-image install-camunda chart-infos cpu-info.txt pods await-zeebe deploy-models rebalance-leaders ${chaosTarget} benchmark await-benchmark +all: chart sync-image install-camunda chart-infos cpu-info.txt pods await-zeebe await-gateway deploy-models rebalance-leaders ${chaosTarget} benchmark await-benchmark #all: chart sync-image install-camunda chart-infos cpu-info.txt pods await-zeebe deploy-models rebalance-leaders ${chaosTarget} await-elasticsearch await-webapps benchmark await-benchmark .PHONY: clean clean: clean-benchmark clean-job-deploy-models uninstall-camunda -ifeq ($(OS),Windows_NT) +# Set the timeout command based on the OS +# On MacOS `brew install coreutils` provides `gtimeout` +TIMEOUT_CMD := $(shell uname | grep -q 'Darwin' && echo 'gtimeout' || echo 'timeout') + +ifeq ("$(OS)", "Windows_NT") root ?= $(CURDIR)/../../../../camunda-8-helm-profiles else root ?= $(shell pwd)/../../../../camunda-8-helm-profiles @@ -36,9 +44,9 @@ include $(root)/metrics/metrics.mk # for Grafana URL .PHONY: sync-image sync-image: - ifneq '${engine.image}' 'camunda/zeebe' +ifneq ("$(engineImage)", "camunda/zeebe") docker manifest inspect gcr.io/camunda-researchanddevelopment/falko-zeebe:${engine.engineVersion} || docker buildx imagetools create --tag gcr.io/camunda-researchanddevelopment/falko-zeebe:${engine.engineVersion} gcr.io/zeebe-io/zeebe:${engine.engineVersion} - endif +endif #: Deploy BPMN and DMN models using zbctl in a k8s job deploy-models: copy-models job-deploy-models logs-job-deploy-models clean-job-deploy-models @@ -79,26 +87,99 @@ deploy-chaos-gateway: deploy-chaos-broker: kubectl apply -f chaos-network-brokers.yaml -n $(namespace) +.PHONY: generate-benchmark-configs # Generate benchmark configs with dynamic gateway IPs +generate-benchmark-configs: +ifeq ("$(gatewayPinning)", "TRUE") + @echo "Generating benchmark configs with dynamic gateway IPs..." + @GATEWAY_IPS=$$(kubectl get pods -l app.kubernetes.io/component=zeebe-gateway -n $(namespace) -o jsonpath='{.items[*].status.podIP}' | tr ' ' '\n' | sort); \ + IP_COUNT=$$(echo "$$GATEWAY_IPS" | wc -l); \ + if [ $$IP_COUNT -eq 0 ]; then \ + echo "No gateway IPs found!"; \ + exit 1; \ + fi; \ + i=0; \ + while [ $$i -lt $(gatewayReplicas) ]; do \ + IP_INDEX=$$((i % IP_COUNT)); \ + GATEWAY_IP=$$(echo "$$GATEWAY_IPS" | sed -n "$$((IP_INDEX + 1))p"); \ + export GATEWAY_IP=$$GATEWAY_IP; \ + export BENCHMARK_ID=$$i; \ + envsubst < benchmark-gateway-template.yaml > benchmark-gateway-$$i.yaml; \ + echo "Generated benchmark-gateway-$$i.yaml with IP $$GATEWAY_IP (round-robin $$IP_INDEX)"; \ + i=$$((i + 1)); \ + done +endif + .PHONY: benchmark -benchmark: namespace copy-models - kubectl create configmap payload --from-file=$(payload)=models/$(payload) -n $(namespace) +benchmark: namespace copy-models generate-benchmark-configs +ifeq ("$(gatewayPinning)", "TRUE") + kubectl apply -f camunda-zeebe-gateway-headless.yaml -n $(namespace) +endif + kubectl create configmap payload --from-file=$(payload)=models/$(payload) -n $(namespace) # kubectl create configmap msg-scenario --from-file=msg-scenario.json=models/msg-scenario.json -n $(namespace) kubectl create configmap msg-scenario --from-file=msg-scenario.json=models/$(scenario) -n $(namespace) +ifeq ("$(gatewayPinning)", "TRUE") + i=0; \ + while [ $$i -lt $(gatewayReplicas) ]; do \ + kubectl apply -f benchmark-gateway-$$i.yaml -n $(namespace); \ + i=$$((i + 1)); \ + done +else kubectl apply -f $(benchmark) -n $(namespace) - kubectl expose deployment benchmark --type=ClusterIP --name=benchmark-svc --port=8088 --target-port=8088 - kubectl apply -f benchmark-service-monitor.yaml -n $(namespace) +# kubectl expose deployment benchmark --type=ClusterIP --name=benchmark-svc --port=8088 --target-port=8088 -n $(namespace) + kubectl apply -f benchmark-service-monitor.yaml -n $(namespace) +endif # TODO automatically switch/generate benchmark config .PHONY: clean-benchmark clean-benchmark: - -kubectl delete -f $(benchmark) -n $(namespace) - -kubectl delete configmap payload -n $(namespace) - -kubectl delete configmap msg-scenario -n $(namespace) +ifeq ("$(gatewayPinning)", "TRUE") + i=0; \ + while [ $$i -lt $(gatewayReplicas) ]; do \ + kubectl delete -f benchmark-gateway-$$i.yaml -n $(namespace); \ + i=$$((i + 1)); \ + done +else + -kubectl delete service benchmark-svc -n $(namespace) + -kubectl delete -f $(benchmark) -n $(namespace) +endif + -kubectl delete configmap payload -n $(namespace) + -kubectl delete configmap msg-scenario -n $(namespace) .PHONY: await-benchmark await-benchmark: - kubectl wait --for=condition=Ready pod -n $(namespace) -l app=benchmark --timeout=900s +ifeq ("$(gatewayPinning)", "TRUE") + i=0; \ + while [ $$i -lt $(gatewayReplicas) ]; do \ + kubectl rollout status --watch statefulset/benchmark-gateway-$$i --timeout=900s -n $(namespace); \ + i=$$((i + 1)); \ + done +else +# kubectl rollout status --watch deployment/benchmark --timeout=900s -n $(namespace) + kubectl rollout status --watch statefulset/benchmark --timeout=900s -n $(namespace) +endif .PHONY: logs-benchmark logs-benchmark: - kubectl logs -f -l app=benchmark -n $(namespace) \ No newline at end of file +ifeq ("$(gatewayPinning)", "TRUE") + selector=""; i=0; \ + while [ $$i -lt $(gatewayReplicas) ]; do \ + if [ -z "$$selector" ]; then selector="benchmark-gateway-$$i"; else selector="$$selector,benchmark-gateway-$$i"; fi; \ + i=$$((i+1)); \ + done; \ + kubectl logs -f -l "app in ($$selector)" -n $(namespace) +else + kubectl logs -f -l app=benchmark -n $(namespace) +endif + +.PHONY: logs-benchmark-timed +logs-benchmark-timed: +ifeq ("$(gatewayPinning)", "TRUE") +# stern benchmark-gateway-[0-$(gatewayReplicas)] -n $(namespace) + @$(TIMEOUT_CMD) ${loadGeneratorStarter.runDuration}s stern benchmark-gateway-[0-$(gatewayReplicas)] -n $(namespace) \ + || { exit_code=$$?; [ $$exit_code -eq 124 ] && echo "${loadGeneratorStarter.runDuration}s timeout reached. Disconnecting from the bechmark logs." || exit $$exit_code; } +else +# @$(TIMEOUT_CMD) 1200s kubectl logs -f -l "app in (benchmark-gateway-1,benchmark-gateway-2,benchmark-gateway-3,benchmark-gateway-4)" --max-log-requests=8 -n $(namespace) \ + || { exit_code=$$?; [ $$exit_code -eq 124 ] && echo "1200s timeout reached. Disconnecting from the bechmark logs." || exit $$exit_code; } + @$(TIMEOUT_CMD) ${loadGeneratorStarter.runDuration}s stern benchmark -n $(namespace) \ + || { exit_code=$$?; [ $$exit_code -eq 124 ] && echo "${loadGeneratorStarter.runDuration}s timeout reached. Disconnecting from the bechmark logs." || exit $$exit_code; } +endif diff --git a/src/main/resources/benchmarktemplates/benchmark-gateway-template.yaml b/src/main/resources/benchmarktemplates/benchmark-gateway-template.yaml new file mode 100644 index 0000000..4b08323 --- /dev/null +++ b/src/main/resources/benchmarktemplates/benchmark-gateway-template.yaml @@ -0,0 +1,74 @@ +# config for https://github.com/camunda-community-hub/camunda-8-benchmark +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: benchmark-gateway-${BENCHMARK_ID} + labels: + app: benchmark-gateway-${BENCHMARK_ID} +spec: + selector: + matchLabels: + app: benchmark-gateway-${BENCHMARK_ID} + serviceName: benchmark-gateway-${BENCHMARK_ID} + replicas: 1 + template: + metadata: +# annotations: +# linkerd.io/inject: enabled +# config.linkerd.io/opaque-ports: "26500" + labels: + app: benchmark-gateway-${BENCHMARK_ID} +# istio.io/dataplane-mode: ambient + spec: + containers: + - name: benchmark + image: camundacommunityhub/camunda-8-benchmark:main + imagePullPolicy: Always + env: + - name: BENCHMARK_STARTER_ID + value: benchmark-gateway-${BENCHMARK_ID} + - name: JAVA_OPTIONS + value: >- + -Dcamunda.client.mode=selfmanaged + -Dcamunda.client.zeebe.grpc-address=http://${GATEWAY_IP}:26500 + -Dcamunda.client.zeebe.prefer-rest-over-grpc=false + -Dcamunda.client.zeebe.request-timeout=600s + -Dcamunda.client.zeebe.defaults.poll-interval=1ms + -Dcamunda.client.zeebe.default-job-worker-stream-enabled=true + -Dcamunda.client.worker.defaults.stream-enabled=true + -Dbenchmark.startProcesses=true + -Dbenchmark.bpmnProcessId=${loadGeneratorStarter.processModel} + -Dbenchmark.startPiPerSecond=${loadGeneratorStarter.startThroughput} + -Dbenchmark.startPiIncreaseFactor=${loadGeneratorStarter.startPiIncreaseFactor} + -Dbenchmark.startRateAdjustmentStrategy=${loadGeneratorStarter.rateAdjustmentStrategy} + -Dbenchmark.fixedBackOffDelay=${jobWorker.fixedBackOffDelay} + -Dbenchmark.maxBackpressurePercentage=1.0 + -Dbenchmark.startWorkers=true + -Dbenchmark.jobType=benchmark-task + -Dbenchmark.multipleJobTypes=${jobWorker.numberOfJobTypes} + -Dbenchmark.taskCompletionDelay=${jobWorker.jobDuration} + -Dbenchmark.payloadPath=file:/${loadGeneratorStarter.payload} + -Dbenchmark.autoDeployProcess=false + -Dbenchmark.warmupPhaseDurationMillis=${loadGeneratorStarter.rampUpTime}000 + -Dbenchmark.enablePartitionPinning=${jobWorker.partitionPinning} + -Dbenchmark.partitionCount=${engine.partitions} + -Dbenchmark.numberOfStarters=${loadGeneratorStarter.starterReplicas} +# -Dcamunda.client.zeebe.grpc-address=http://camunda-zeebe-gateway-grpc:26500 +# -Dzeebe.client.worker.max-jobs-active=${jobWorker.maxJobsActive} +# -Dzeebe.client.worker.threads=${jobWorker.numberOfThreads} + resources: + limits: + cpu: ${engine.vcpus} + memory: ${engine.ram}Gi + requests: + cpu: 5 + memory: 5Gi + volumeMounts: + - name: payload + mountPath: ${loadGeneratorStarter.payload} + subPath: ${loadGeneratorStarter.payload} + readOnly: true + volumes: + - name: payload + configMap: + name: payload diff --git a/src/main/resources/benchmarktemplates/benchmark.yaml b/src/main/resources/benchmarktemplates/benchmark.yaml index 0a3ae24..ed325ae 100644 --- a/src/main/resources/benchmarktemplates/benchmark.yaml +++ b/src/main/resources/benchmarktemplates/benchmark.yaml @@ -1,6 +1,6 @@ # config for https://github.com/camunda-community-hub/camunda-8-benchmark apiVersion: apps/v1 -kind: Deployment +kind: StatefulSet metadata: name: benchmark labels: @@ -9,21 +9,30 @@ spec: selector: matchLabels: app: benchmark + serviceName: benchmark replicas: ${loadGeneratorStarter.starterReplicas} template: metadata: +# annotations: +# linkerd.io/inject: enabled +# config.linkerd.io/opaque-ports: "26500" labels: app: benchmark +# istio.io/dataplane-mode: ambient spec: containers: - name: benchmark image: camundacommunityhub/camunda-8-benchmark:main imagePullPolicy: Always env: + - name: BENCHMARK_STARTER_ID + valueFrom: + fieldRef: + fieldPath: metadata.name - name: JAVA_OPTIONS value: >- -Dcamunda.client.mode=selfmanaged - -Dcamunda.client.zeebe.grpc-address=http://camunda-zeebe-gateway:26500 + -Dcamunda.client.zeebe.grpc-address=http://camunda-zeebe-gateway:26500 -Dcamunda.client.zeebe.prefer-rest-over-grpc=false -Dcamunda.client.zeebe.request-timeout=600s -Dcamunda.client.zeebe.defaults.poll-interval=1ms @@ -41,8 +50,13 @@ spec: -Dbenchmark.multipleJobTypes=${jobWorker.numberOfJobTypes} -Dbenchmark.taskCompletionDelay=${jobWorker.jobDuration} -Dbenchmark.payloadPath=file:/${loadGeneratorStarter.payload} + -Dbenchmark.fixedBackOffDelay=${jobWorker.fixedBackOffDelay} -Dbenchmark.autoDeployProcess=false -Dbenchmark.warmupPhaseDurationMillis=${loadGeneratorStarter.rampUpTime}000 + -Dbenchmark.enablePartitionPinning=${jobWorker.partitionPinning} + -Dbenchmark.partitionCount=${engine.partitions} + -Dbenchmark.numberOfStarters=${loadGeneratorStarter.starterReplicas} +# -Dcamunda.client.zeebe.grpc-address=http://camunda-zeebe-gateway-grpc:26500 # -Dzeebe.client.worker.max-jobs-active=${jobWorker.maxJobsActive} # -Dzeebe.client.worker.threads=${jobWorker.numberOfThreads} resources: @@ -50,8 +64,8 @@ spec: cpu: ${engine.vcpus} memory: ${engine.ram}Gi requests: - cpu: 1 - memory: 1Gi + cpu: 100m + memory: 512Mi volumeMounts: - name: payload mountPath: ${loadGeneratorStarter.payload} diff --git a/src/main/resources/benchmarktemplates/camunda-values.yaml b/src/main/resources/benchmarktemplates/camunda-values.yaml index 7bf225d..a3a555d 100644 --- a/src/main/resources/benchmarktemplates/camunda-values.yaml +++ b/src/main/resources/benchmarktemplates/camunda-values.yaml @@ -6,15 +6,39 @@ global: elasticsearch: enabled: ${elasticSearch.enabled} + ingress: + enabled: ${optimize.enabled} + className: nginx + host: "falko-benchmark.gke.c8sm.com" + tls: + enabled: true + secretName: "tls-secret" identity: auth: # Disabling the Identity authentication # it will fall back to basic-auth: demo/demo as default user enabled: ${optimize.enabled} + publicIssuerUrl: "https://falko-benchmark.gke.c8sm.com/auth/realms/camunda-platform" + identity: + recirectUrl: "https://falko-benchmark.gke.c8sm.com/identity" + operate: + redirectUrl: "https://falko-benchmark.gke.c8sm.com/operate" + tasklist: + redirectUrl: "https://falko-benchmark.gke.c8sm.com/tasklist" + optimize: + redirectUrl: "https://falko-benchmark.gke.c8sm.com/optimize" + connectors: + redirectUrl: "https://falko-benchmark.gke.c8sm.com/connectors" + webModeler: + redirectUrl: "https://falko-benchmark.gke.c8sm.com/modeler" + console: + redirectUrl: "https://falko-benchmark.gke.c8sm.com/console" zeebe: -# nodeSelector: -# cloud.google.com/gke-nodepool: pool-${engine.machineType} + # affinity: + # podAntiAffinity: null + nodeSelector: + cloud.google.com/gke-nodepool: pool-${engine.machineType} # alpha.eksctl.io/nodegroup-name: c7g-8xlarge resources: requests: @@ -31,12 +55,35 @@ zeebe: persistenceType: ${engine.diskType} pvcSize: ${engine.diskSize}Gi pvcStorageClassName: ${engine.fileSystem} + # initContainers: + # - name: init-exporter-job-worker + # image: curlimages/curl:latest + # command: ["/bin/sh", "-c"] + # args: [ + # "curl -L https://camunda.jfrog.io/artifactory/camunda-bpm-community-extensions/org/camunda/community/extension/zeebe/exporter/jobworker/zeebe-embedded-job-worker/0.0.10/zeebe-embedded-job-worker-0.0.10.jar -o /exporters/zeebe-embedded-job-worker-0.0.10.jar; ls -al /exporters" + # ] + # securityContext: + # runAsUser: 1001 + # runAsNonRoot: true + # volumeMounts: + # - name: exporters + # mountPath: /exporters/ + # extraVolumes: + # - name: exporters-writable + # emptyDir: {} + # extraVolumeMounts: + # - name: exporters-writable + # mountPath: /usr/local/zeebe/exporters/ env: + # - name: ZEEBE_BROKER_EXPORTERS_JOBWORKER_JARPATH + # value: exporters/zeebe-embedded-job-worker-0.0.10.jar + # - name: ZEEBE_BROKER_EXPORTERS_JOBWORKER_CLASSNAME + # value: org.camunda.community.extension.zeebe.exporter.jobworker.EmbeddedJobWorker # comment out ZEEBE_BROKER_EXPORTERS_ELASTICSEARCH* when disabling Elasticsearch - - name: ZEEBE_BROKER_EXPORTERS_ELASTICSEARCH_ARGS_INDEX_NUMBEROFSHARDS - value: "${elasticSearch.shardsPerIndex}" - - name: ZEEBE_BROKER_EXPORTERS_ELASTICSEARCH_ARGS_INDEX_NUMBEROFREPLICAS - value: "1" + #- name: ZEEBE_BROKER_EXPORTERS_ELASTICSEARCH_ARGS_INDEX_NUMBEROFSHARDS + # value: "${elasticSearch.shardsPerIndex}" + #- name: ZEEBE_BROKER_EXPORTERS_ELASTICSEARCH_ARGS_INDEX_NUMBEROFREPLICAS + # value: "1" # Export to separate Elasticsearch for Optimize #- name: ZEEBE_BROKER_EXPORTERS_ELASTICSEARCHOPTIMIZE_CLASSNAME # value: io.camunda.zeebe.exporter.ElasticsearchExporter @@ -103,15 +150,36 @@ zeebe: fieldPath: metadata.namespace zeebeGateway: + # affinity: + # podAntiAffinity: null + contextPath: "/rest" +# podAnnotations: +# linkerd.io/inject: enabled +# config.linkerd.io/opaque-ports: "26500" + ingress: + grpc: + enabled: true + host: "falko-benchmark.gke.c8sm.com" + tls: + enabled: true + secretName: "tls-secret" + rest: + enabled: true + host: "falko-benchmark.gke.c8sm.com" + path: "/rest" + tls: + enabled: true + secretName: "tls-secret" replicas: ${gateway.replicas} -# nodeSelector: -# cloud.google.com/gke-nodepool: pool-${engine.machineType} + nodeSelector: + cloud.google.com/gke-nodepool: pool-${engine.machineType} +# alpha.eksctl.io/nodegroup-name: c7g-8xlarge resources: requests: cpu: ${gateway.vcpus} memory: ${gateway.ram}Gi limits: - cpu: ${gateway.vcpus} + cpu: null memory: ${gateway.ram}Gi env: - name: ZEEBE_GATEWAY_MONITORING_ENABLED @@ -134,6 +202,9 @@ zeebeGateway: operate: enabled: ${operate.enabled} + contextPath: "/operate" + nodeSelector: + cloud.google.com/gke-nodepool: pool-${engine.machineType} resources: requests: cpu: ${operate.vcpus} @@ -160,10 +231,86 @@ operate: value: "1" - name: CAMUNDA_OPERATE_ZEEBEELASTICSEARCH_BATCHSIZE value: "${operate.importPageSize}" + configuration: | + server: + servlet: + context-path: "/operate" + management: + server: + base-path: "/operate" + + spring: + servlet: + multipart: + max-file-size: "10MB" + max-request-size: "10MB" + profiles: + active: "identity-auth" + security: + oauth2: + resourceserver: + jwt: + issuer-uri: "http://camunda-keycloak:80/auth/realms/camunda-platform" + jwk-set-uri: "http://camunda-keycloak:80/auth/realms/camunda-platform/protocol/openid-connect/certs" + camunda: + identity: + clientId: "operate" + audience: "operate-api" + + # Camunda Database configuration + camunda.database: + type: elasticsearch + # Cluster name + clusterName: elasticsearch + # Elasticsearch full url + url: "http://camunda-elasticsearch:9200" + + # Operate configuration file + camunda.operate: + identity: + redirectRootUrl: "https://falko-benchmark.gke.c8sm.com" + + # ELS instance to store Operate data + elasticsearch: + # Cluster name + clusterName: elasticsearch + # Host + host: camunda-elasticsearch + # Transport port + port: 9200 + # Elasticsearch full url + url: "http://camunda-elasticsearch:9200" + numberOfShards: ${elasticSearch.shardsPerIndex} + numberOfReplicas: 1 + # ELS instance to export Zeebe data to + zeebeElasticsearch: + # Cluster name + clusterName: elasticsearch + # Host + host: camunda-elasticsearch + # Transport port + port: 9200 + # Index prefix, configured in Zeebe Elasticsearch exporter + prefix: zeebe-record + # Elasticsearch full url + url: "http://camunda-elasticsearch:9200" + # Zeebe instance + zeebe: + # Gateway address + gatewayAddress: "camunda-zeebe-gateway:26500" + logging: + level: + ROOT: INFO + io.camunda.operate: INFO + #Spring Boot Actuator endpoints to be exposed + management.endpoints.web.exposure.include: health,info,conditions,configprops,prometheus,loggers,usage-metrics,backups + + tasklist: enabled: ${operate.enabled} - #nodeSelector: - # cloud.google.com/gke-nodepool: pool-${engine.machineType} + contextPath: "/tasklist" + nodeSelector: + cloud.google.com/gke-nodepool: pool-${engine.machineType} resources: requests: cpu: ${operate.vcpus} @@ -193,9 +340,10 @@ tasklist: optimize: enabled: ${optimize.enabled} + contextPath: "/optimize" partitionCount: "${engine.partitions}" - #nodeSelector: - # cloud.google.com/gke-nodepool: pool-${engine.machineType} + nodeSelector: + cloud.google.com/gke-nodepool: pool-${engine.machineType} resources: requests: cpu: "${optimize.vcpus}" @@ -232,9 +380,13 @@ optimize: connectors: enabled: false + inbound: + mode: oauth + contextPath: "/connectors" webModeler: enabled: false + contextPath: "/modeler" image: pullSecrets: - name: camunda-docker-registry @@ -247,12 +399,20 @@ webModeler: console: enabled: ${optimize.enabled} + contextPath: "/console" identity: enabled: ${optimize.enabled} + contextPath: "/identity" + fullURL: "https://falko-benchmark.gke.c8sm.com/identity" identityKeycloak: enabled: ${optimize.enabled} + extraEnvVars: + - name: KEYCLOAK_PROXY_ADDRESS_FORWARDING + value: "true" + - name: KEYCLOAK_FRONTEND_URL + value: "https://falko-benchmark.gke.c8sm.com/auth" postgresql: primary: resourcesPreset: micro @@ -270,8 +430,8 @@ elasticsearch: master: masterOnly: ${elasticSearch.masterOnly} replicaCount: ${elasticSearch.clusterSizeMaster} - #nodeSelector: - # cloud.google.com/gke-nodepool: pool-${engine.machineType} + nodeSelector: + cloud.google.com/gke-nodepool: pool-${engine.machineType} resources: requests: cpu: ${elasticSearch.vcpusMasterRequests} @@ -285,8 +445,8 @@ elasticsearch: storageClass: ${elasticSearch.fileSystem} data: replicaCount: ${elasticSearch.clusterSizeData} - #nodeSelector: - # cloud.google.com/gke-nodepool: pool-${engine.machineType} + nodeSelector: + cloud.google.com/gke-nodepool: pool-${engine.machineType} resources: requests: cpu: ${elasticSearch.vcpusDataRequests} diff --git a/src/main/resources/benchmarktemplates/camunda-zeebe-gateway-headless.yaml b/src/main/resources/benchmarktemplates/camunda-zeebe-gateway-headless.yaml new file mode 100644 index 0000000..2f379fe --- /dev/null +++ b/src/main/resources/benchmarktemplates/camunda-zeebe-gateway-headless.yaml @@ -0,0 +1,25 @@ +apiVersion: v1 +kind: Service +metadata: + name: camunda-zeebe-gateway-headless + labels: + app: camunda-platform + app.kubernetes.io/name: camunda-platform + app.kubernetes.io/instance: camunda + app.kubernetes.io/part-of: camunda-platform + app.kubernetes.io/component: zeebe-gateway +spec: + clusterIP: None # This makes it a headless service + type: ClusterIP + selector: + app: camunda-platform + app.kubernetes.io/name: camunda-platform + app.kubernetes.io/instance: camunda + app.kubernetes.io/part-of: camunda-platform + app.kubernetes.io/component: zeebe-gateway + ports: + - name: grpc + port: 26500 + targetPort: 26500 + protocol: TCP + appProtocol: kubernetes.io/h2c diff --git a/src/main/resources/benchmarktemplates/job-cpu-info.yaml b/src/main/resources/benchmarktemplates/job-cpu-info.yaml index 4e8b248..95f4ce6 100644 --- a/src/main/resources/benchmarktemplates/job-cpu-info.yaml +++ b/src/main/resources/benchmarktemplates/job-cpu-info.yaml @@ -11,8 +11,8 @@ spec: batch.kubernetes.io/job-name: job-cpu-info job-name: job-cpu-info spec: -# nodeSelector: -# cloud.google.com/gke-nodepool: pool-${engine.machineType} + nodeSelector: + cloud.google.com/gke-nodepool: pool-${engine.machineType} # alpha.eksctl.io/nodegroup-name: c7g-8xlarge containers: - command: diff --git a/src/main/resources/benchmarktemplates/run.sh b/src/main/resources/benchmarktemplates/run.sh index 7889f26..002621a 100644 --- a/src/main/resources/benchmarktemplates/run.sh +++ b/src/main/resources/benchmarktemplates/run.sh @@ -1,2 +1,8 @@ echo "Waiting $((${loadGeneratorStarter.runDuration}/60)) minutes for completion of benchmark run ${testCaseName} ..." -sleep ${loadGeneratorStarter.runDuration} + +if [[ "$(uname)" != "Darwin" ]] || command -v gtimeout >/dev/null 2>&1; then + make logs-benchmark-timed +else + echo "gtimeout is not installed. Using sleep instead. To install gtimeout run: brew install coreutils" + sleep ${loadGeneratorStarter.runDuration} +fi \ No newline at end of file