Skip to content

Commit 12af6af

Browse files
authored
Upgrade jmx-exporter to v1.5.0 (#189)
Build jmx_exporter from upstream prometheus releases. Currently the jmx-javaagent docker image is built from https://github.com/amuraru/jmx_exporter In order to bring that dependency in-tree here, this PR extends kafka image(s) release (kafka-*) action to also build a docker image directly from https://github.com/prometheus/jmx_exporter The image is published to: https://github.com/adobe/koperator/pkgs/container/koperator%2Fjmx-javaagent
1 parent 5704d5a commit 12af6af

File tree

16 files changed

+324
-62
lines changed

16 files changed

+324
-62
lines changed

.github/workflows/publish-kafka-docker.yml

Lines changed: 91 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,9 @@ env:
1313
# GitHub Container Registry
1414
GHCR_REGISTRY: ghcr.io
1515
# github.repository as <account>/<repo>
16-
GHCR_IMAGE_NAME: ${{ github.repository }}/kafka
17-
# Docker Hub image name
16+
GHCR_KAFKA_IMAGE_NAME: ${{ github.repository }}/kafka
17+
GHCR_JMX_IMAGE_NAME: ${{ github.repository }}/jmx-javaagent
18+
# Docker Hub image name (only for Kafka)
1819
DOCKERHUB_IMAGE: adobe/kafka
1920

2021
jobs:
@@ -34,7 +35,7 @@ jobs:
3435
run: |
3536
echo "built_at=$(date --rfc-3339=date)" >> $GITHUB_OUTPUT
3637
37-
build-kafka:
38+
build-and-push-kafka-image:
3839
runs-on: ubuntu-latest
3940
needs: prepare
4041
permissions:
@@ -47,40 +48,122 @@ jobs:
4748
uses: docker/setup-qemu-action@v3
4849
- name: Set up Docker Buildx
4950
uses: docker/setup-buildx-action@v3
51+
- name: Extract Kafka version from tag
52+
id: kafka_version
53+
run: |
54+
KAFKA_VERSION=$(echo "${{ needs.prepare.outputs.tag }}" | sed 's/kafka-//')
55+
echo "kafka_version=$KAFKA_VERSION" >> $GITHUB_OUTPUT
56+
- name: Check if Kafka image already exists
57+
id: image_exists
58+
run: |
59+
IMAGE_NAME="${{ env.GHCR_REGISTRY }}/${{ env.GHCR_KAFKA_IMAGE_NAME }}:${{ steps.kafka_version.outputs.kafka_version }}"
60+
echo "Checking if image exists: $IMAGE_NAME"
61+
62+
# Try to inspect the manifest using docker
63+
if docker manifest inspect "$IMAGE_NAME" > /dev/null 2>&1; then
64+
echo "exists=true" >> $GITHUB_OUTPUT
65+
echo "Image $IMAGE_NAME already exists, skipping build"
66+
else
67+
echo "exists=false" >> $GITHUB_OUTPUT
68+
echo "Image $IMAGE_NAME does not exist, will build"
69+
fi
5070
- name: Login to DockerHub Registry
51-
if: startsWith(github.ref, 'refs/tags/')
71+
if: startsWith(github.ref, 'refs/tags/') && github.repository == 'adobe/koperator' && steps.image_exists.outputs.exists == 'false'
5272
uses: docker/login-action@v3
5373
with:
5474
username: ${{ secrets.DOCKER_USERNAME }}
5575
password: ${{ secrets.DOCKER_PASSWORD }}
5676
- name: Log into GitHub Container Registry ${{ env.GHCR_REGISTRY }}
57-
if: startsWith(github.ref, 'refs/tags/')
77+
if: startsWith(github.ref, 'refs/tags/') && steps.image_exists.outputs.exists == 'false'
5878
uses: docker/login-action@v3
5979
with:
6080
registry: ${{ env.GHCR_REGISTRY }}
6181
username: ${{ github.actor }}
6282
password: ${{ secrets.GITHUB_TOKEN }}
6383
- name: Extract Docker metadata
84+
if: steps.image_exists.outputs.exists == 'false'
6485
id: meta
6586
uses: docker/metadata-action@v5
6687
with:
6788
images: |
68-
${{ env.DOCKERHUB_IMAGE }}
69-
${{ env.GHCR_REGISTRY }}/${{ env.GHCR_IMAGE_NAME }}
89+
${{ github.repository == 'adobe/koperator' && env.DOCKERHUB_IMAGE || '' }}
90+
${{ env.GHCR_REGISTRY }}/${{ env.GHCR_KAFKA_IMAGE_NAME }}
7091
tags: |
7192
type=match,pattern=kafka-(.*),group=1
7293
labels: |
7394
org.opencontainers.image.description=Apache Kafka with OpenJDK 21
7495
- name: Build and push kafka image
96+
if: startsWith(github.ref, 'refs/tags/') && steps.image_exists.outputs.exists == 'false'
7597
uses: docker/build-push-action@v6
7698
with:
7799
context: docker/kafka
78100
platforms: linux/amd64,linux/arm64
79-
push: ${{ startsWith(github.ref, 'refs/tags/') }}
101+
push: true
80102
tags: ${{ steps.meta.outputs.tags }}
81103
labels: ${{ steps.meta.outputs.labels }}
82104
annotations: ${{ steps.meta.outputs.annotations }}
83105
build-args: |
84106
VERSION=${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.version'] }}
85107
BUILT_AT=${{ needs.prepare.outputs.built_at }}
86108
COMMIT=${{ github.sha }}
109+
110+
build-and-push-jmx-agent-image:
111+
runs-on: ubuntu-latest
112+
needs: prepare
113+
permissions:
114+
contents: read
115+
packages: write
116+
steps:
117+
- name: Checkout code
118+
uses: actions/checkout@v5
119+
- name: Set up QEMU
120+
uses: docker/setup-qemu-action@v3
121+
- name: Set up Docker Buildx
122+
uses: docker/setup-buildx-action@v3
123+
- name: Extract JMX exporter version from Dockerfile
124+
id: jmx_exporter_version
125+
run: |
126+
JMX_VERSION=$(grep "ARG JMX_EXPORTER_VERSION=" docker/jmx_exporter/Dockerfile | cut -d'=' -f2 | cut -d' ' -f1)
127+
echo "jmx_version=$JMX_VERSION" >> $GITHUB_OUTPUT
128+
- name: Check if JMX exporter image already exists
129+
id: image_exists
130+
run: |
131+
IMAGE_NAME="${{ env.GHCR_REGISTRY }}/${{ env.GHCR_JMX_IMAGE_NAME }}:${{ steps.jmx_exporter_version.outputs.jmx_version }}"
132+
echo "Checking if image exists: $IMAGE_NAME"
133+
134+
# Try to inspect the manifest using docker
135+
if docker manifest inspect "$IMAGE_NAME" > /dev/null 2>&1; then
136+
echo "exists=true" >> $GITHUB_OUTPUT
137+
echo "Image $IMAGE_NAME already exists, skipping build"
138+
else
139+
echo "exists=false" >> $GITHUB_OUTPUT
140+
echo "Image $IMAGE_NAME does not exist, will build"
141+
fi
142+
- name: Log into GitHub Container Registry ${{ env.GHCR_REGISTRY }}
143+
if: startsWith(github.ref, 'refs/tags/') && steps.image_exists.outputs.exists == 'false'
144+
uses: docker/login-action@v3
145+
with:
146+
registry: ${{ env.GHCR_REGISTRY }}
147+
username: ${{ github.actor }}
148+
password: ${{ secrets.GITHUB_TOKEN }}
149+
- name: Extract Docker metadata for JMX exporter
150+
if: steps.image_exists.outputs.exists == 'false'
151+
id: meta
152+
uses: docker/metadata-action@v5
153+
with:
154+
images: |
155+
${{ env.GHCR_REGISTRY }}/${{ env.GHCR_JMX_IMAGE_NAME }}
156+
tags: |
157+
type=raw,value=${{ steps.jmx_exporter_version.outputs.jmx_version }}
158+
labels: |
159+
org.opencontainers.image.description=Prometheus JMX Exporter for Kafka monitoring
160+
- name: Build and push JMX exporter image
161+
if: startsWith(github.ref, 'refs/tags/') && steps.image_exists.outputs.exists == 'false'
162+
uses: docker/build-push-action@v6
163+
with:
164+
context: docker/jmx_exporter
165+
platforms: linux/amd64,linux/arm64
166+
push: true
167+
tags: ${{ steps.meta.outputs.tags }}
168+
labels: ${{ steps.meta.outputs.labels }}
169+
annotations: ${{ steps.meta.outputs.annotations }}

.licensei.toml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ ignored = [
1616
"github.com/rcrowley/go-metrics",
1717
"go.uber.org/atomic",
1818
"go.uber.org/multierr",
19+
"go.uber.org/mock",
1920
"emperror.dev/errors",
2021
"emperror.dev/emperror",
2122
"go.uber.org/zap",

api/assets/kafka/jmx-exporter.yml

Lines changed: 25 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,34 @@
1+
httpServer:
2+
threads:
3+
minimum: 5
4+
maximum: 10
5+
keepAliveTime: 120
16
lowercaseOutputName: true
27
rules:
3-
# Special cases and very specific rules
4-
# Export kraft current state metrics
8+
# Export Kraft metrics
9+
## Export Kraft current-state count metrics
510
- pattern: 'kafka.server<type=raft-metrics><>current-state: (.+)'
611
name: kafka_server_raft_metrics_current_state_$1
712
type: GAUGE
813
value: 1
14+
## MetadataLoader count metrics as COUNTERs
15+
- pattern: kafka.server<type=MetadataLoader><>([A-Za-z]+Count)
16+
name: kafka_server_metadataloader_$1_total
17+
type: COUNTER
18+
## Broker metadata error count metrics as COUNTERs
19+
- pattern: kafka.server<type=broker-metadata-metrics><>(metadata-(load|apply)-error-count)
20+
name: kafka_server_broker_metadata_metrics_$1_total
21+
type: COUNTER
22+
## KafkaController ActiveBrokerCount as GAUGE (current state, not cumulative)
23+
- pattern: kafka.controller<type=KafkaController><>ActiveBrokerCount
24+
name: kafka_controller_kafkacontroller_ActiveBrokerCount
25+
type: GAUGE
26+
## KafkaController count metrics as COUNTERs
27+
- pattern: kafka.controller<type=KafkaController><>([A-Za-z]+Count)
28+
name: kafka_controller_kafkacontroller_$1_total
29+
type: COUNTER
30+
# END: Export Kraft metrics
31+
932
- pattern: 'kafka.server<type=(app-info), id=(\d+)><>(Version): ([-.~+\w\d]+)'
1033
name: kafka_server_$1_$3
1134
type: COUNTER

api/v1beta1/kafkacluster_types.go

Lines changed: 9 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -53,10 +53,13 @@ const (
5353
IsControllerNodeKey = "isControllerNode"
5454

5555
// DefaultCruiseControlImage is the default CC image used when users don't specify it in CruiseControlConfig.Image
56-
DefaultCruiseControlImage = "adobe/cruise-control:3.0.3-adbe-20250804"
56+
DefaultCruiseControlImage = "adobe/cruise-control:3.0.3-adbe-20250804" // renovate: datasource=docker depName=adobe/cruise-control
5757

5858
// DefaultKafkaImage is the default Kafka image used when users don't specify it in KafkaClusterSpec.ClusterImage
59-
DefaultKafkaImage = "ghcr.io/adobe/koperator/kafka:2.13-3.9.1"
59+
DefaultKafkaImage = "ghcr.io/adobe/koperator/kafka:2.13-3.9.1" // renovate: datasource=docker depName=ghcr.io/adobe/koperator/kafka
60+
61+
// DefaultMonitorImage is the default JMX monitor image used when users don't specify it in MonitoringConfig.JmxImage
62+
DefaultMonitorImage = "ghcr.io/adobe/koperator/jmx-javaagent:1.4.0" // renovate: datasource=docker depName=ghcr.io/adobe/koperator/jmx-javaagent
6063

6164
// ControllerNodeProcessRole represents the node is a controller node
6265
ControllerNodeProcessRole = "controller"
@@ -91,7 +94,7 @@ const (
9194
defaultEnvoyLimitResourceMemory = "100Mi"
9295

9396
// KafkaClusterDeployment.spec.template.spec.container["envoy"].image
94-
defaultEnvoyImage = "envoyproxy/envoy:v1.22.2"
97+
defaultEnvoyImage = "envoyproxy/envoy:v1.22.2" // renovate: datasource=docker depName=envoyproxy/envoy
9598

9699
/* Broker Config */
97100

@@ -110,7 +113,7 @@ const (
110113
/* Cruise Control Config */
111114

112115
// CruiseControlDeployment.spec.template.spec.container["%s-cruisecontrol"].image
113-
defaultCruiseControlImage = "adobe/cruise-control:3.0.3-adbe-20250804"
116+
defaultCruiseControlImage = "adobe/cruise-control:3.0.3-adbe-20250804" // renovate: datasource=docker depName=adobe/cruise-control
114117

115118
// CruiseControlDeployment.spec.template.spec.container["%s-cruisecontrol"].resources
116119
defaultCruiseControlRequestResourceCpu = "200m"
@@ -126,7 +129,7 @@ const (
126129
defaultKafkaClusterK8sClusterDomain = "cluster.local"
127130

128131
// KafkaBroker.spec.container["kafka"].image
129-
defaultKafkaImage = "ghcr.io/adobe/koperator/kafka:2.13-3.9.1"
132+
defaultKafkaImage = "ghcr.io/adobe/koperator/kafka:2.13-3.9.1" // renovate: datasource=docker depName=ghcr.io/adobe/koperator/kafka
130133

131134
/* Istio Ingress Config */
132135

@@ -143,10 +146,6 @@ const (
143146

144147
/* Monitor Config */
145148

146-
// KafkaBrokerPod.spec.initContainer[jmx-exporter].image
147-
// kafkaClusterDeployment.spec.template.spec.initContainer["jmx-exporter"].image
148-
defaultMonitorImage = "ghcr.io/amuraru/jmx-javaagent:0.19.2"
149-
150149
// KafkaBrokerPod.spec.initContainer["jmx-exporter"].command
151150
// kafkaClusterDeployment.spec.template.spec.initContainer["jmx-exporter"].command
152151
defaultMonitorPathToJar = "/jmx_prometheus_javaagent.jar"
@@ -1296,7 +1295,7 @@ func (mConfig *MonitoringConfig) GetImage() string {
12961295
if mConfig.JmxImage != "" {
12971296
return mConfig.JmxImage
12981297
}
1299-
return defaultMonitorImage
1298+
return DefaultMonitorImage
13001299
}
13011300

13021301
// GetPathToJar returns the path in the used Image for Prometheus JMX exporter

config/samples/banzaicloud_v1beta1_kafkacluster.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -708,7 +708,7 @@ spec:
708708
# monitoringConfig describes the monitoring related configs
709709
#monitoringConfig:
710710
# jmxImage describes the used prometheus jmx exporter agent container
711-
# jmxImage: "ghcr.io/amuraru/jmx-javaagent:0.19.2"
711+
# jmxImage: "ghcr.io/adobe/koperator/jmx-javaagent:1.4.0"
712712
# pathToJar describes the path to the jar file in the given image
713713
# pathToJar: "/jmx_prometheus_javaagent.jar"
714714
# kafkaJMXExporterConfig describes jmx exporter config for Kafka

config/samples/kraft/simplekafkacluster_kraft.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ metadata:
77
spec:
88
kRaft: true
99
monitoringConfig:
10-
jmxImage: "ghcr.io/amuraru/jmx-javaagent:0.19.2"
10+
jmxImage: "ghcr.io/adobe/koperator/jmx-javaagent:1.4.0"
1111
headlessServiceEnabled: true
1212
propagateLabels: false
1313
oneBrokerPerNode: false

config/samples/simplekafkacluster.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ metadata:
77
spec:
88
kRaft: false
99
monitoringConfig:
10-
jmxImage: "ghcr.io/amuraru/jmx-javaagent:0.19.2"
10+
jmxImage: "ghcr.io/adobe/koperator/jmx-javaagent:1.4.0"
1111
headlessServiceEnabled: true
1212
zkAddresses:
1313
- "zookeeper-server-client.zookeeper:2181"

config/samples/simplekafkacluster_with_contour.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ metadata:
66
name: kafka
77
spec:
88
monitoringConfig:
9-
jmxImage: "ghcr.io/amuraru/jmx-javaagent:0.19.2"
9+
jmxImage: "ghcr.io/adobe/koperator/jmx-javaagent:1.4.0"
1010
headlessServiceEnabled: true
1111
zkAddresses:
1212
- "zookeeper-server-client.zookeeper:2181"

controllers/tests/kafkacluster_controller_cruisecontrol_test.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -277,7 +277,7 @@ func expectCruiseControlDeployment(ctx context.Context, kafkaCluster *v1beta1.Ka
277277
Expect(deployment.Spec.Template.Spec.InitContainers).To(HaveLen(1))
278278
initContainer := deployment.Spec.Template.Spec.InitContainers[0]
279279
Expect(initContainer.Name).To(Equal("jmx-exporter"))
280-
Expect(initContainer.Image).To(Equal("ghcr.io/amuraru/jmx-javaagent:0.19.2"))
280+
Expect(initContainer.Image).To(Equal(v1beta1.DefaultMonitorImage))
281281
Expect(initContainer.Command).To(Equal([]string{"cp", "/jmx_prometheus_javaagent.jar", "/opt/jmx-exporter/jmx_prometheus.jar"}))
282282
Expect(initContainer.VolumeMounts).To(ConsistOf(corev1.VolumeMount{
283283
Name: "jmx-jar-data",
@@ -292,7 +292,7 @@ func expectCruiseControlDeployment(ctx context.Context, kafkaCluster *v1beta1.Ka
292292
Expect(container.Lifecycle).NotTo(BeNil())
293293
Expect(container.Lifecycle.PreStop).NotTo(BeNil())
294294
Expect(container.Lifecycle.PreStop.Exec).NotTo(BeNil())
295-
Expect(container.Image).To(Equal("adobe/cruise-control:3.0.3-adbe-20250804"))
295+
Expect(container.Image).To(Equal(v1beta1.DefaultCruiseControlImage))
296296
Expect(container.Ports).To(ConsistOf(
297297
corev1.ContainerPort{
298298
ContainerPort: 8090,

docker/jmx_exporter/Dockerfile

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
ARG JMX_EXPORTER_VERSION=1.5.0 # renovate: datasource=github-releases depName=prometheus/jmx_exporter
2+
3+
FROM maven:3-amazoncorretto-21 AS build
4+
ARG JMX_EXPORTER_VERSION
5+
6+
# Install wget to download the release tarball
7+
RUN yum install -y wget && yum clean all
8+
9+
# Download and extract the JMX exporter release
10+
RUN wget https://github.com/prometheus/jmx_exporter/archive/refs/tags/${JMX_EXPORTER_VERSION}.tar.gz \
11+
-O /tmp/jmx_exporter.tar.gz && \
12+
tar -xzf /tmp/jmx_exporter.tar.gz -C /tmp && \
13+
mv /tmp/jmx_exporter-${JMX_EXPORTER_VERSION} /src && \
14+
rm /tmp/jmx_exporter.tar.gz
15+
16+
WORKDIR /src
17+
RUN mvn -B -Dmaven.javadoc.skip=true -Dskip.javadoc=true \
18+
-Dmaven.compiler.source=21 -Dmaven.compiler.target=21 -Dmaven.compiler.release=21 \
19+
clean package -pl jmx_prometheus_javaagent -am
20+
21+
FROM alpine:latest
22+
ARG JMX_EXPORTER_VERSION
23+
COPY --from=build /src/jmx_prometheus_javaagent/target/jmx_prometheus_javaagent-${JMX_EXPORTER_VERSION}.jar /opt/jmx_exporter/
24+
RUN ln -s /opt/jmx_exporter/jmx_prometheus_javaagent-${JMX_EXPORTER_VERSION}.jar /jmx_prometheus_javaagent.jar
25+
CMD ["/bin/sh"]

0 commit comments

Comments
 (0)