diff --git a/.github/workflows/publish-kafka-docker.yml b/.github/workflows/publish-kafka-docker.yml index 3e4408a33..1ef3fd7bc 100644 --- a/.github/workflows/publish-kafka-docker.yml +++ b/.github/workflows/publish-kafka-docker.yml @@ -13,8 +13,9 @@ env: # GitHub Container Registry GHCR_REGISTRY: ghcr.io # github.repository as / - GHCR_IMAGE_NAME: ${{ github.repository }}/kafka - # Docker Hub image name + GHCR_KAFKA_IMAGE_NAME: ${{ github.repository }}/kafka + GHCR_JMX_IMAGE_NAME: ${{ github.repository }}/jmx-javaagent + # Docker Hub image name (only for Kafka) DOCKERHUB_IMAGE: adobe/kafka jobs: @@ -34,7 +35,7 @@ jobs: run: | echo "built_at=$(date --rfc-3339=date)" >> $GITHUB_OUTPUT - build-kafka: + build-and-push-kafka-image: runs-on: ubuntu-latest needs: prepare permissions: @@ -47,36 +48,57 @@ jobs: uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + - name: Extract Kafka version from tag + id: kafka_version + run: | + KAFKA_VERSION=$(echo "${{ needs.prepare.outputs.tag }}" | sed 's/kafka-//') + echo "kafka_version=$KAFKA_VERSION" >> $GITHUB_OUTPUT + - name: Check if Kafka image already exists + id: image_exists + run: | + IMAGE_NAME="${{ env.GHCR_REGISTRY }}/${{ env.GHCR_KAFKA_IMAGE_NAME }}:${{ steps.kafka_version.outputs.kafka_version }}" + echo "Checking if image exists: $IMAGE_NAME" + + # Try to inspect the manifest using docker + if docker manifest inspect "$IMAGE_NAME" > /dev/null 2>&1; then + echo "exists=true" >> $GITHUB_OUTPUT + echo "Image $IMAGE_NAME already exists, skipping build" + else + echo "exists=false" >> $GITHUB_OUTPUT + echo "Image $IMAGE_NAME does not exist, will build" + fi - name: Login to DockerHub Registry - if: startsWith(github.ref, 'refs/tags/') + if: startsWith(github.ref, 'refs/tags/') && github.repository == 'adobe/koperator' && steps.image_exists.outputs.exists == 'false' uses: docker/login-action@v3 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Log into GitHub Container Registry ${{ env.GHCR_REGISTRY }} - if: startsWith(github.ref, 'refs/tags/') + if: startsWith(github.ref, 'refs/tags/') && steps.image_exists.outputs.exists == 'false' uses: docker/login-action@v3 with: registry: ${{ env.GHCR_REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Extract Docker metadata + if: steps.image_exists.outputs.exists == 'false' id: meta uses: docker/metadata-action@v5 with: images: | - ${{ env.DOCKERHUB_IMAGE }} - ${{ env.GHCR_REGISTRY }}/${{ env.GHCR_IMAGE_NAME }} + ${{ github.repository == 'adobe/koperator' && env.DOCKERHUB_IMAGE || '' }} + ${{ env.GHCR_REGISTRY }}/${{ env.GHCR_KAFKA_IMAGE_NAME }} tags: | type=match,pattern=kafka-(.*),group=1 labels: | org.opencontainers.image.description=Apache Kafka with OpenJDK 21 - name: Build and push kafka image + if: startsWith(github.ref, 'refs/tags/') && steps.image_exists.outputs.exists == 'false' uses: docker/build-push-action@v6 with: context: docker/kafka platforms: linux/amd64,linux/arm64 - push: ${{ startsWith(github.ref, 'refs/tags/') }} + push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} annotations: ${{ steps.meta.outputs.annotations }} @@ -84,3 +106,64 @@ jobs: VERSION=${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.version'] }} BUILT_AT=${{ needs.prepare.outputs.built_at }} COMMIT=${{ github.sha }} + + build-and-push-jmx-agent-image: + runs-on: ubuntu-latest + needs: prepare + permissions: + contents: read + packages: write + steps: + - name: Checkout code + uses: actions/checkout@v5 + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Extract JMX exporter version from Dockerfile + id: jmx_exporter_version + run: | + JMX_VERSION=$(grep "ARG JMX_EXPORTER_VERSION=" docker/jmx_exporter/Dockerfile | cut -d'=' -f2 | cut -d' ' -f1) + echo "jmx_version=$JMX_VERSION" >> $GITHUB_OUTPUT + - name: Check if JMX exporter image already exists + id: image_exists + run: | + IMAGE_NAME="${{ env.GHCR_REGISTRY }}/${{ env.GHCR_JMX_IMAGE_NAME }}:${{ steps.jmx_exporter_version.outputs.jmx_version }}" + echo "Checking if image exists: $IMAGE_NAME" + + # Try to inspect the manifest using docker + if docker manifest inspect "$IMAGE_NAME" > /dev/null 2>&1; then + echo "exists=true" >> $GITHUB_OUTPUT + echo "Image $IMAGE_NAME already exists, skipping build" + else + echo "exists=false" >> $GITHUB_OUTPUT + echo "Image $IMAGE_NAME does not exist, will build" + fi + - name: Log into GitHub Container Registry ${{ env.GHCR_REGISTRY }} + if: startsWith(github.ref, 'refs/tags/') && steps.image_exists.outputs.exists == 'false' + uses: docker/login-action@v3 + with: + registry: ${{ env.GHCR_REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Extract Docker metadata for JMX exporter + if: steps.image_exists.outputs.exists == 'false' + id: meta + uses: docker/metadata-action@v5 + with: + images: | + ${{ env.GHCR_REGISTRY }}/${{ env.GHCR_JMX_IMAGE_NAME }} + tags: | + type=raw,value=${{ steps.jmx_exporter_version.outputs.jmx_version }} + labels: | + org.opencontainers.image.description=Prometheus JMX Exporter for Kafka monitoring + - name: Build and push JMX exporter image + if: startsWith(github.ref, 'refs/tags/') && steps.image_exists.outputs.exists == 'false' + uses: docker/build-push-action@v6 + with: + context: docker/jmx_exporter + platforms: linux/amd64,linux/arm64 + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + annotations: ${{ steps.meta.outputs.annotations }} diff --git a/.licensei.toml b/.licensei.toml index a4b98fc04..cb629f7a4 100644 --- a/.licensei.toml +++ b/.licensei.toml @@ -16,6 +16,7 @@ ignored = [ "github.com/rcrowley/go-metrics", "go.uber.org/atomic", "go.uber.org/multierr", + "go.uber.org/mock", "emperror.dev/errors", "emperror.dev/emperror", "go.uber.org/zap", diff --git a/api/assets/kafka/jmx-exporter.yml b/api/assets/kafka/jmx-exporter.yml index c2451e575..68d762b7d 100644 --- a/api/assets/kafka/jmx-exporter.yml +++ b/api/assets/kafka/jmx-exporter.yml @@ -1,11 +1,34 @@ +httpServer: + threads: + minimum: 5 + maximum: 10 + keepAliveTime: 120 lowercaseOutputName: true rules: - # Special cases and very specific rules - # Export kraft current state metrics + # Export Kraft metrics + ## Export Kraft current-state count metrics - pattern: 'kafka.server<>current-state: (.+)' name: kafka_server_raft_metrics_current_state_$1 type: GAUGE value: 1 + ## MetadataLoader count metrics as COUNTERs + - pattern: kafka.server<>([A-Za-z]+Count) + name: kafka_server_metadataloader_$1_total + type: COUNTER + ## Broker metadata error count metrics as COUNTERs + - pattern: kafka.server<>(metadata-(load|apply)-error-count) + name: kafka_server_broker_metadata_metrics_$1_total + type: COUNTER + ## KafkaController ActiveBrokerCount as GAUGE (current state, not cumulative) + - pattern: kafka.controller<>ActiveBrokerCount + name: kafka_controller_kafkacontroller_ActiveBrokerCount + type: GAUGE + ## KafkaController count metrics as COUNTERs + - pattern: kafka.controller<>([A-Za-z]+Count) + name: kafka_controller_kafkacontroller_$1_total + type: COUNTER + # END: Export Kraft metrics + - pattern: 'kafka.server<>(Version): ([-.~+\w\d]+)' name: kafka_server_$1_$3 type: COUNTER diff --git a/api/v1beta1/kafkacluster_types.go b/api/v1beta1/kafkacluster_types.go index 806b55118..557fca3d4 100644 --- a/api/v1beta1/kafkacluster_types.go +++ b/api/v1beta1/kafkacluster_types.go @@ -53,10 +53,13 @@ const ( IsControllerNodeKey = "isControllerNode" // DefaultCruiseControlImage is the default CC image used when users don't specify it in CruiseControlConfig.Image - DefaultCruiseControlImage = "adobe/cruise-control:3.0.3-adbe-20250804" + DefaultCruiseControlImage = "adobe/cruise-control:3.0.3-adbe-20250804" // renovate: datasource=docker depName=adobe/cruise-control // DefaultKafkaImage is the default Kafka image used when users don't specify it in KafkaClusterSpec.ClusterImage - DefaultKafkaImage = "ghcr.io/adobe/koperator/kafka:2.13-3.9.1" + DefaultKafkaImage = "ghcr.io/adobe/koperator/kafka:2.13-3.9.1" // renovate: datasource=docker depName=ghcr.io/adobe/koperator/kafka + + // DefaultMonitorImage is the default JMX monitor image used when users don't specify it in MonitoringConfig.JmxImage + DefaultMonitorImage = "ghcr.io/adobe/koperator/jmx-javaagent:1.4.0" // renovate: datasource=docker depName=ghcr.io/adobe/koperator/jmx-javaagent // ControllerNodeProcessRole represents the node is a controller node ControllerNodeProcessRole = "controller" @@ -91,7 +94,7 @@ const ( defaultEnvoyLimitResourceMemory = "100Mi" // KafkaClusterDeployment.spec.template.spec.container["envoy"].image - defaultEnvoyImage = "envoyproxy/envoy:v1.22.2" + defaultEnvoyImage = "envoyproxy/envoy:v1.22.2" // renovate: datasource=docker depName=envoyproxy/envoy /* Broker Config */ @@ -110,7 +113,7 @@ const ( /* Cruise Control Config */ // CruiseControlDeployment.spec.template.spec.container["%s-cruisecontrol"].image - defaultCruiseControlImage = "adobe/cruise-control:3.0.3-adbe-20250804" + defaultCruiseControlImage = "adobe/cruise-control:3.0.3-adbe-20250804" // renovate: datasource=docker depName=adobe/cruise-control // CruiseControlDeployment.spec.template.spec.container["%s-cruisecontrol"].resources defaultCruiseControlRequestResourceCpu = "200m" @@ -126,7 +129,7 @@ const ( defaultKafkaClusterK8sClusterDomain = "cluster.local" // KafkaBroker.spec.container["kafka"].image - defaultKafkaImage = "ghcr.io/adobe/koperator/kafka:2.13-3.9.1" + defaultKafkaImage = "ghcr.io/adobe/koperator/kafka:2.13-3.9.1" // renovate: datasource=docker depName=ghcr.io/adobe/koperator/kafka /* Istio Ingress Config */ @@ -143,10 +146,6 @@ const ( /* Monitor Config */ - // KafkaBrokerPod.spec.initContainer[jmx-exporter].image - // kafkaClusterDeployment.spec.template.spec.initContainer["jmx-exporter"].image - defaultMonitorImage = "ghcr.io/amuraru/jmx-javaagent:0.19.2" - // KafkaBrokerPod.spec.initContainer["jmx-exporter"].command // kafkaClusterDeployment.spec.template.spec.initContainer["jmx-exporter"].command defaultMonitorPathToJar = "/jmx_prometheus_javaagent.jar" @@ -1296,7 +1295,7 @@ func (mConfig *MonitoringConfig) GetImage() string { if mConfig.JmxImage != "" { return mConfig.JmxImage } - return defaultMonitorImage + return DefaultMonitorImage } // GetPathToJar returns the path in the used Image for Prometheus JMX exporter diff --git a/config/samples/banzaicloud_v1beta1_kafkacluster.yaml b/config/samples/banzaicloud_v1beta1_kafkacluster.yaml index c0d5ffe07..09d784675 100644 --- a/config/samples/banzaicloud_v1beta1_kafkacluster.yaml +++ b/config/samples/banzaicloud_v1beta1_kafkacluster.yaml @@ -708,7 +708,7 @@ spec: # monitoringConfig describes the monitoring related configs #monitoringConfig: # jmxImage describes the used prometheus jmx exporter agent container - # jmxImage: "ghcr.io/amuraru/jmx-javaagent:0.19.2" + # jmxImage: "ghcr.io/adobe/koperator/jmx-javaagent:1.4.0" # pathToJar describes the path to the jar file in the given image # pathToJar: "/jmx_prometheus_javaagent.jar" # kafkaJMXExporterConfig describes jmx exporter config for Kafka diff --git a/config/samples/kraft/simplekafkacluster_kraft.yaml b/config/samples/kraft/simplekafkacluster_kraft.yaml index 2b73503ba..793066691 100644 --- a/config/samples/kraft/simplekafkacluster_kraft.yaml +++ b/config/samples/kraft/simplekafkacluster_kraft.yaml @@ -7,7 +7,7 @@ metadata: spec: kRaft: true monitoringConfig: - jmxImage: "ghcr.io/amuraru/jmx-javaagent:0.19.2" + jmxImage: "ghcr.io/adobe/koperator/jmx-javaagent:1.4.0" headlessServiceEnabled: true propagateLabels: false oneBrokerPerNode: false diff --git a/config/samples/simplekafkacluster.yaml b/config/samples/simplekafkacluster.yaml index 740c95718..d890f8551 100644 --- a/config/samples/simplekafkacluster.yaml +++ b/config/samples/simplekafkacluster.yaml @@ -7,7 +7,7 @@ metadata: spec: kRaft: false monitoringConfig: - jmxImage: "ghcr.io/amuraru/jmx-javaagent:0.19.2" + jmxImage: "ghcr.io/adobe/koperator/jmx-javaagent:1.4.0" headlessServiceEnabled: true zkAddresses: - "zookeeper-server-client.zookeeper:2181" diff --git a/config/samples/simplekafkacluster_with_contour.yaml b/config/samples/simplekafkacluster_with_contour.yaml index d63143c26..d8f4b8838 100644 --- a/config/samples/simplekafkacluster_with_contour.yaml +++ b/config/samples/simplekafkacluster_with_contour.yaml @@ -6,7 +6,7 @@ metadata: name: kafka spec: monitoringConfig: - jmxImage: "ghcr.io/amuraru/jmx-javaagent:0.19.2" + jmxImage: "ghcr.io/adobe/koperator/jmx-javaagent:1.4.0" headlessServiceEnabled: true zkAddresses: - "zookeeper-server-client.zookeeper:2181" diff --git a/controllers/tests/kafkacluster_controller_cruisecontrol_test.go b/controllers/tests/kafkacluster_controller_cruisecontrol_test.go index 3d9f6c620..23a88a2c1 100644 --- a/controllers/tests/kafkacluster_controller_cruisecontrol_test.go +++ b/controllers/tests/kafkacluster_controller_cruisecontrol_test.go @@ -277,7 +277,7 @@ func expectCruiseControlDeployment(ctx context.Context, kafkaCluster *v1beta1.Ka Expect(deployment.Spec.Template.Spec.InitContainers).To(HaveLen(1)) initContainer := deployment.Spec.Template.Spec.InitContainers[0] Expect(initContainer.Name).To(Equal("jmx-exporter")) - Expect(initContainer.Image).To(Equal("ghcr.io/amuraru/jmx-javaagent:0.19.2")) + Expect(initContainer.Image).To(Equal(v1beta1.DefaultMonitorImage)) Expect(initContainer.Command).To(Equal([]string{"cp", "/jmx_prometheus_javaagent.jar", "/opt/jmx-exporter/jmx_prometheus.jar"})) Expect(initContainer.VolumeMounts).To(ConsistOf(corev1.VolumeMount{ Name: "jmx-jar-data", @@ -292,7 +292,7 @@ func expectCruiseControlDeployment(ctx context.Context, kafkaCluster *v1beta1.Ka Expect(container.Lifecycle).NotTo(BeNil()) Expect(container.Lifecycle.PreStop).NotTo(BeNil()) Expect(container.Lifecycle.PreStop.Exec).NotTo(BeNil()) - Expect(container.Image).To(Equal("adobe/cruise-control:3.0.3-adbe-20250804")) + Expect(container.Image).To(Equal(v1beta1.DefaultCruiseControlImage)) Expect(container.Ports).To(ConsistOf( corev1.ContainerPort{ ContainerPort: 8090, diff --git a/docker/jmx_exporter/Dockerfile b/docker/jmx_exporter/Dockerfile new file mode 100644 index 000000000..3a250d537 --- /dev/null +++ b/docker/jmx_exporter/Dockerfile @@ -0,0 +1,25 @@ +ARG JMX_EXPORTER_VERSION=1.5.0 # renovate: datasource=github-releases depName=prometheus/jmx_exporter + +FROM maven:3-amazoncorretto-21 AS build +ARG JMX_EXPORTER_VERSION + +# Install wget to download the release tarball +RUN yum install -y wget && yum clean all + +# Download and extract the JMX exporter release +RUN wget https://github.com/prometheus/jmx_exporter/archive/refs/tags/${JMX_EXPORTER_VERSION}.tar.gz \ + -O /tmp/jmx_exporter.tar.gz && \ + tar -xzf /tmp/jmx_exporter.tar.gz -C /tmp && \ + mv /tmp/jmx_exporter-${JMX_EXPORTER_VERSION} /src && \ + rm /tmp/jmx_exporter.tar.gz + +WORKDIR /src +RUN mvn -B -Dmaven.javadoc.skip=true -Dskip.javadoc=true \ + -Dmaven.compiler.source=21 -Dmaven.compiler.target=21 -Dmaven.compiler.release=21 \ + clean package -pl jmx_prometheus_javaagent -am + +FROM alpine:latest +ARG JMX_EXPORTER_VERSION +COPY --from=build /src/jmx_prometheus_javaagent/target/jmx_prometheus_javaagent-${JMX_EXPORTER_VERSION}.jar /opt/jmx_exporter/ +RUN ln -s /opt/jmx_exporter/jmx_prometheus_javaagent-${JMX_EXPORTER_VERSION}.jar /jmx_prometheus_javaagent.jar +CMD ["/bin/sh"] diff --git a/docker/kafka/Dockerfile b/docker/kafka/Dockerfile index aefdc21cc..3e3a0d6df 100644 --- a/docker/kafka/Dockerfile +++ b/docker/kafka/Dockerfile @@ -1,37 +1,42 @@ +ARG scala_version=2.13 +ARG kafka_version=3.9.1 # renovate: datasource=github-tags depName=apache/kafka +ARG java_version=21 + FROM alpine:latest@sha256:4b7ce07002c69e8f3d704a9c5d6fd3053be500b7f1c69fc0d80990c2ad8dd412 AS kafka_dist -ARG scala_version=2.13 -ARG kafka_version=3.9.1 +ARG scala_version +ARG kafka_version ARG kafka_distro_base_url=https://downloads.apache.org/kafka ENV kafka_distro=kafka_$scala_version-$kafka_version.tgz ENV kafka_distro_asc=$kafka_distro.asc -RUN apk add --no-cache gnupg - WORKDIR /var/tmp -RUN wget -q $kafka_distro_base_url/$kafka_version/$kafka_distro -RUN wget -q $kafka_distro_base_url/$kafka_version/$kafka_distro_asc -RUN wget -q $kafka_distro_base_url/KEYS - -RUN gpg --import KEYS -RUN gpg --verify $kafka_distro_asc $kafka_distro - -RUN tar -xzf $kafka_distro -RUN rm -r kafka_$scala_version-$kafka_version/bin/windows +# Download, verify, extract and clean up Kafka in single layer +RUN apk add --no-cache gnupg wget && \ + wget -q $kafka_distro_base_url/$kafka_version/$kafka_distro && \ + wget -q $kafka_distro_base_url/$kafka_version/$kafka_distro_asc && \ + wget -q $kafka_distro_base_url/KEYS && \ + gpg --import KEYS && \ + gpg --verify $kafka_distro_asc $kafka_distro && \ + tar -xzf $kafka_distro && \ + rm -rf kafka_$scala_version-$kafka_version/bin/windows \ + kafka_$scala_version-$kafka_version/site-docs \ + $kafka_distro $kafka_distro_asc KEYS && \ + find kafka_$scala_version-$kafka_version -name "*.bat" -delete # backported from https://github.com/docker-library/openjdk/blob/master/18/jdk/slim-bullseye/Dockerfile FROM debian:bullseye-slim@sha256:52927eff8153b563244f98cdc802ba97918afcdf67f9e4867cbf1f7afb3d147b -ARG scala_version=2.13 -ARG kafka_version=3.9.1 +ARG scala_version +ARG kafka_version +ARG java_version ARG VERSION ARG BUILT_AT ARG COMMIT - RUN set -eux; \ apt-get update; \ apt-get install -y --no-install-recommends \ @@ -39,24 +44,24 @@ RUN set -eux; \ ; \ rm -rf /var/lib/apt/lists/* -ENV JAVA_HOME /usr/local/openjdk-21 -ENV PATH $JAVA_HOME/bin:$PATH +ENV JAVA_HOME=/usr/local/openjdk-${java_version} +ENV PATH=$JAVA_HOME/bin:$PATH # Default to UTF-8 file.encoding -ENV LANG C.UTF-8 -ENV JAVA_VERSION 21 +ENV LANG=C.UTF-8 +ENV JAVA_VERSION=${java_version} RUN set -eux; \ \ arch="$(dpkg --print-architecture)"; \ case "$arch" in \ 'amd64') \ - downloadUrl='https://download.oracle.com/java/21/latest/jdk-21_linux-x64_bin.tar.gz'; \ - downloadSha256='https://download.oracle.com/java/21/latest/jdk-21_linux-x64_bin.tar.gz.sha256'; \ + downloadUrl="https://download.oracle.com/java/${java_version}/latest/jdk-${java_version}_linux-x64_bin.tar.gz"; \ + downloadSha256="https://download.oracle.com/java/${java_version}/latest/jdk-${java_version}_linux-x64_bin.tar.gz.sha256"; \ ;; \ 'arm64') \ - downloadUrl='https://download.oracle.com/java/21/latest/jdk-21_linux-aarch64_bin.tar.gz'; \ - downloadSha256='https://download.oracle.com/java/21/latest/jdk-21_linux-aarch64_bin.tar.gz.sha256'; \ + downloadUrl="https://download.oracle.com/java/${java_version}/latest/jdk-${java_version}_linux-aarch64_bin.tar.gz"; \ + downloadSha256="https://download.oracle.com/java/${java_version}/latest/jdk-${java_version}_linux-aarch64_bin.tar.gz.sha256"; \ ;; \ *) echo >&2 "error: unsupported architecture: '$arch'"; exit 1 ;; \ esac; \ @@ -81,6 +86,9 @@ RUN set -eux; \ ; \ rm openjdk.tgz*; \ \ +# Remove JMODs (not needed for runtime, only for jlink custom images) + rm -rf "$JAVA_HOME/jmods"; \ + \ apt-mark auto '.*' > /dev/null; \ [ -z "$savedAptMark" ] || apt-mark manual $savedAptMark > /dev/null; \ apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false; \ @@ -117,18 +125,25 @@ ENV KAFKA_VERSION=$kafka_version \ ENV PATH=${PATH}:${KAFKA_HOME}/bin -RUN mkdir ${KAFKA_HOME} && apt-get update && apt-get -y upgrade && apt-get install curl -y && apt-get clean +# Install minimal dependencies, setup Kafka, and clean up in single layer +RUN apt-get update && \ + apt-get install -y --no-install-recommends curl && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* && \ + mkdir -p ${KAFKA_HOME}/libs/extensions COPY --from=kafka_dist /var/tmp/kafka_$scala_version-$kafka_version ${KAFKA_HOME} COPY opt/kafka/config/log4j.properties ${KAFKA_HOME}/config/log4j.properties - -RUN chmod a+x ${KAFKA_HOME}/bin/*.sh -RUN chmod g+rwX ${KAFKA_HOME} && mkdir -p ${KAFKA_HOME}/libs/extensions && chmod g+rwX ${KAFKA_HOME}/libs/extensions +# Optimize Kafka installation: remove unnecessary files and set permissions +RUN chmod a+x ${KAFKA_HOME}/bin/*.sh && \ + chmod g+rwX ${KAFKA_HOME} ${KAFKA_HOME}/libs/extensions && \ + rm -rf ${KAFKA_HOME}/bin/windows ${KAFKA_HOME}/site-docs && \ + find ${KAFKA_HOME} -name "*.bat" -delete # Add metadata labels LABEL org.opencontainers.image.title="Apache Kafka" -LABEL org.opencontainers.image.description="Apache Kafka with OpenJDK 21" +LABEL org.opencontainers.image.description="Apache Kafka with Oracle JDK ${java_version}" LABEL org.opencontainers.image.vendor="Adobe" LABEL org.opencontainers.image.licenses="Apache-2.0" LABEL org.opencontainers.image.source="https://github.com/adobe/koperator" diff --git a/docs/benchmarks/infrastructure/kafka_202001_3brokers.yaml b/docs/benchmarks/infrastructure/kafka_202001_3brokers.yaml index df0df665a..7c8928ea6 100644 --- a/docs/benchmarks/infrastructure/kafka_202001_3brokers.yaml +++ b/docs/benchmarks/infrastructure/kafka_202001_3brokers.yaml @@ -7,7 +7,7 @@ metadata: spec: kRaft: false monitoringConfig: - jmxImage: "ghcr.io/amuraru/jmx-javaagent:0.19.2" + jmxImage: "ghcr.io/adobe/koperator/jmx-javaagent:1.5.0" pathToJar: "/jmx_prometheus_javaagent.jar" kafkaJMXExporterConfig: |2 lowercaseOutputName: true diff --git a/renovate.json b/renovate.json index 84d79a819..2d09b6a5b 100644 --- a/renovate.json +++ b/renovate.json @@ -19,13 +19,37 @@ { "customType": "regex", "managerFilePatterns": [ - "/Makefile/" + "/Makefile/", + "/Dockerfile$/" ], "matchStrings": [ "(?\\w+)_VERSION\\s*=\\s*(?[^\\s]+)\\s*#\\s*renovate:\\s*datasource=(?[^\\s]+)\\s*depName=(?[^\\s]+)" ], "datasourceTemplate": "{{datasource}}", "depNameTemplate": "{{packageName}}" + }, + { + "customType": "regex", + "managerFilePatterns": [ + "tests/e2e/versions.go" + ], + "matchStrings": [ + "\\s*(?\\w+)\\s*=\\s*\"(?[^\"]+)\"\\s*//\\s*renovate:\\s*datasource=(?[^\\s]+)\\s*depName=(?[^\\s]+)(?:\\s*registryUrl=(?[^\\s]+))?" + ], + "datasourceTemplate": "{{datasource}}", + "depNameTemplate": "{{packageName}}", + "registryUrlTemplate": "{{#if registryUrl}}{{{registryUrl}}}{{/if}}" + }, + { + "customType": "regex", + "managerFilePatterns": [ + "api/v1beta1/kafkacluster_types.go" + ], + "matchStrings": [ + "\\s*(?\\w+)\\s*=\\s*\"(?[^\"]+)\"\\s*//\\s*renovate:\\s*datasource=(?[^\\s]+)\\s*depName=(?[^\\s]+)" + ], + "datasourceTemplate": "{{datasource}}", + "depNameTemplate": "{{packageName}}" } ], "packageRules": [ @@ -42,6 +66,52 @@ ], "pinDigests": true }, + { + "groupName": "config samples docker images", + "groupSlug": "config-samples-docker", + "matchFilePatterns": [ + "config/samples/**", + "docs/benchmarks/infrastructure/**", + "tests/e2e/templates/**" + ], + "matchManagers": [ + "helm-values" + ], + "matchUpdateTypes": [ + "major", + "minor", + "patch" + ] + }, + { + "groupName": "kafka docker images (major)", + "groupSlug": "kafka-docker-major", + "matchFilePatterns": [ + "docker/**" + ], + "matchManagers": [ + "dockerfile", + "custom.regex" + ], + "matchUpdateTypes": [ + "major" + ] + }, + { + "groupName": "kafka docker images (minor/patch)", + "groupSlug": "kafka-docker-minor-patch", + "matchFilePatterns": [ + "docker/**" + ], + "matchManagers": [ + "dockerfile", + "custom.regex" + ], + "matchUpdateTypes": [ + "minor", + "patch" + ] + }, { "groupName": "all major dependencies", "groupSlug": "all-major", @@ -55,6 +125,12 @@ "docker-compose", "custom.regex", "helm-values" + ], + "excludeFilePatterns": [ + "docker/**", + "config/samples/**", + "docs/benchmarks/infrastructure/**", + "tests/e2e/templates/**" ] }, { @@ -73,6 +149,12 @@ "custom.regex", "helm-values", "docker" + ], + "excludeFilePatterns": [ + "docker/**", + "config/samples/**", + "docs/benchmarks/infrastructure/**", + "tests/e2e/templates/**" ] } ] diff --git a/tests/e2e/global.go b/tests/e2e/global.go index 5228e01e4..8102543cc 100644 --- a/tests/e2e/global.go +++ b/tests/e2e/global.go @@ -27,7 +27,7 @@ var ( certManagerHelmDescriptor = helmDescriptor{ Repository: "https://charts.jetstack.io", ChartName: "cert-manager", - ChartVersion: "v1.18.2", + ChartVersion: CertManagerVersion, ReleaseName: "cert-manager", Namespace: "cert-manager", SetValues: map[string]string{ @@ -42,7 +42,7 @@ var ( contourIngressControllerHelmDescriptor = helmDescriptor{ Repository: "https://projectcontour.github.io/helm-charts", ChartName: "contour", - ChartVersion: "0.1.0", + ChartVersion: ContourVersion, ReleaseName: "contour", Namespace: "projectcontour", SetValues: map[string]string{ @@ -94,7 +94,7 @@ var ( prometheusOperatorHelmDescriptor = helmDescriptor{ Repository: "https://prometheus-community.github.io/helm-charts", ChartName: "kube-prometheus-stack", - ChartVersion: "77.12.0", + ChartVersion: PrometheusOperatorVersion, ReleaseName: "prometheus-operator", Namespace: "prometheus", SetValues: map[string]string{ @@ -123,13 +123,13 @@ var ( zookeeperOperatorHelmDescriptor = helmDescriptor{ Repository: "", ChartName: "oci://ghcr.io/adobe/helm-charts/zookeeper-operator", - ChartVersion: "0.2.15-adobe-20250923", + ChartVersion: ZookeeperOperatorVersion, ReleaseName: "zookeeper-operator", Namespace: "zookeeper", SetValues: map[string]string{ "crd.create": "false", "image.repository": "ghcr.io/adobe/zookeeper-operator", - "image.tag": "0.2.15-adobe-20250923", + "image.tag": ZookeeperOperatorVersion, }, RemoteCRDPathVersionTemplate: "https://raw.githubusercontent.com/adobe/zookeeper-operator/%s/config/crd/bases/zookeeper.pravega.io_zookeeperclusters.yaml", HelmExtraArguments: map[string][]string{ diff --git a/tests/e2e/versions.go b/tests/e2e/versions.go new file mode 100644 index 000000000..a319f82d1 --- /dev/null +++ b/tests/e2e/versions.go @@ -0,0 +1,32 @@ +// Copyright © 2023 Cisco Systems, Inc. and/or its affiliates +// Copyright 2025 Adobe. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +// Software versions used in e2e tests +// These versions are monitored by Renovate for automatic updates +const ( + // CertManagerVersion is the version of cert-manager Helm chart + CertManagerVersion = "v1.18.2" // renovate: datasource=helm depName=cert-manager registryUrl=https://charts.jetstack.io + + // ContourVersion is the version of Contour ingress controller Helm chart + ContourVersion = "0.1.0" // renovate: datasource=helm depName=contour registryUrl=https://projectcontour.github.io/helm-charts + + // PrometheusOperatorVersion is the version of kube-prometheus-stack Helm chart + PrometheusOperatorVersion = "77.12.0" // renovate: datasource=helm depName=kube-prometheus-stack registryUrl=https://prometheus-community.github.io/helm-charts + + // ZookeeperOperatorVersion is the version of zookeeper-operator + ZookeeperOperatorVersion = "0.2.15-adobe-20250923" // renovate: datasource=docker depName=ghcr.io/adobe/helm-charts/zookeeper-operator +) diff --git a/third_party/github.com/banzaicloud/operator-tools/.licensei.toml b/third_party/github.com/banzaicloud/operator-tools/.licensei.toml index a05ea32df..737de6ec2 100644 --- a/third_party/github.com/banzaicloud/operator-tools/.licensei.toml +++ b/third_party/github.com/banzaicloud/operator-tools/.licensei.toml @@ -19,6 +19,8 @@ ignored = [ "gopkg.in/fsnotify.v1", # BSD-3 "github.com/uber-go/tally", # MIT "github.com/russross/blackfriday", # BSD + "go.uber.org/multierr", # MIT + "emperror.dev/errors", # MIT "github.com/xeipuuv/gojsonpointer", # Apache2 "github.com/xeipuuv/gojsonreference", # Apache2