Skip to content

Commit 7155d51

Browse files
dnwednwe
committed
chore(ci): add kafka 3.6.0 to FVT and versions
- add V3_6_0_0 to Sarama - bump default dockerfile KAFKA_VERSION to 3.6.0 - pin ubi-minimal version and digest and include in dependabot - pin apidiff cmd in that workflow - fix some linter warnings in Dockerfile - add USER 65534:65534 to Dockerfile for nobody user Signed-off-by: Dominic Evans <[email protected]>
1 parent e0c3c62 commit 7155d51

9 files changed

+67
-48
lines changed

‎.github/dependabot.yml

+10
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,15 @@
11
---
22
version: 2
33
updates:
4+
- package-ecosystem: docker
5+
directory: "/Dockerfile.*"
6+
schedule:
7+
interval: daily
8+
labels:
9+
- "ignore-for-release"
10+
commit-message:
11+
prefix: chore(ci)
12+
413
- package-ecosystem: github-actions
514
directory: /
615
open-pull-requests-limit: 2
@@ -10,6 +19,7 @@ updates:
1019
- "ignore-for-release"
1120
commit-message:
1221
prefix: chore(ci)
22+
1323
- package-ecosystem: gomod
1424
directory: /
1525
open-pull-requests-limit: 5

‎.github/workflows/apidiff.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ jobs:
2727
- name: Add GOBIN to PATH
2828
run: echo "$(go env GOPATH)/bin" >>$GITHUB_PATH
2929
- name: Install apidiff cmd
30-
run: go install golang.org/x/exp/cmd/apidiff@latest
30+
run: go install golang.org/x/exp/cmd/apidiff@v0.0.0-20231006140011-7918f672742d
3131
- name: Checkout base code
3232
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
3333
with:

‎.github/workflows/fvt-main.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ jobs:
1717
fail-fast: false
1818
matrix:
1919
go-version: [1.21.x]
20-
kafka-version: [1.0.2, 2.0.1, 2.2.2, 2.6.2, 2.8.2, 3.0.2, 3.3.2, 3.5.1]
20+
kafka-version: [1.0.2, 2.0.1, 2.2.2, 2.6.2, 2.8.2, 3.0.2, 3.3.2, 3.6.0]
2121
include:
2222
- kafka-version: 1.0.2
2323
scala-version: 2.11
@@ -33,7 +33,7 @@ jobs:
3333
scala-version: 2.12
3434
- kafka-version: 3.3.2
3535
scala-version: 2.13
36-
- kafka-version: 3.5.1
36+
- kafka-version: 3.6.0
3737
scala-version: 2.13
3838
uses: ./.github/workflows/fvt.yml
3939
with:

‎.github/workflows/fvt-pr.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -16,13 +16,13 @@ jobs:
1616
fail-fast: false
1717
matrix:
1818
go-version: [1.21.x]
19-
kafka-version: [1.0.2, 2.6.2, 3.5.1]
19+
kafka-version: [1.0.2, 2.6.2, 3.6.0]
2020
include:
2121
- kafka-version: 1.0.2
2222
scala-version: 2.11
2323
- kafka-version: 2.6.2
2424
scala-version: 2.12
25-
- kafka-version: 3.5.1
25+
- kafka-version: 3.6.0
2626
scala-version: 2.13
2727
uses: ./.github/workflows/fvt.yml
2828
with:

‎.github/workflows/fvt.yml

+2-1
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ on:
99
kafka-version:
1010
required: false
1111
type: string
12-
default: 3.5.1
12+
default: 3.6.0
1313
scala-version:
1414
required: false
1515
type: string
@@ -38,6 +38,7 @@ jobs:
3838
builder: ${{ steps.buildx.outputs.name }}
3939
files: docker-compose.yml
4040
load: true
41+
targets: kafka-1
4142
set: |
4243
*.cache-from=type=gha,scope=${{ github.workflow }}
4344
*.cache-to=type=gha,scope=${{ github.workflow }},mode=max

‎Dockerfile.kafka

+15-12
Original file line numberDiff line numberDiff line change
@@ -1,35 +1,36 @@
1-
FROM registry.access.redhat.com/ubi8/ubi-minimal:latest
1+
FROM registry.access.redhat.com/ubi8/ubi-minimal:8.8@sha256:b93deceb59a58588d5b16429fc47f98920f84740a1f2ed6454e33275f0701b59
22

33
USER root
44

5-
RUN microdnf update \
6-
&& microdnf install curl gzip java-11-openjdk-headless tar tzdata-java \
7-
&& microdnf reinstall tzdata \
8-
&& microdnf clean all
5+
RUN microdnf update -y \
6+
&& microdnf install -y curl gzip java-11-openjdk-headless tar tzdata-java \
7+
&& microdnf reinstall -y tzdata \
8+
&& microdnf clean all
99

1010
ENV JAVA_HOME=/usr/lib/jvm/jre-11
1111

1212
# https://docs.oracle.com/javase/7/docs/technotes/guides/net/properties.html
1313
# Ensure Java doesn't cache any dns results
1414
RUN cd /etc/java/java-11-openjdk/*/conf/security \
15-
&& sed -e '/networkaddress.cache.ttl/d' -e '/networkaddress.cache.negative.ttl/d' -i java.security \
16-
&& echo 'networkaddress.cache.ttl=0' >> java.security \
17-
&& echo 'networkaddress.cache.negative.ttl=0' >> java.security
15+
&& sed -e '/networkaddress.cache.ttl/d' -e '/networkaddress.cache.negative.ttl/d' -i java.security \
16+
&& echo 'networkaddress.cache.ttl=0' >> java.security \
17+
&& echo 'networkaddress.cache.negative.ttl=0' >> java.security
1818

1919
ARG SCALA_VERSION="2.13"
20-
ARG KAFKA_VERSION="3.5.1"
20+
ARG KAFKA_VERSION="3.6.0"
2121

22-
# https://github.com/apache/kafka/blob/53eeaad946cd053e9eb1a762972d4efeacb8e4fc/tests/docker/Dockerfile#L65-L69
22+
# https://github.com/apache/kafka/blob/9989b68d0d38c8f1357f78bf9d53a58c1476188d/tests/docker/Dockerfile#L46-L72
2323
ARG KAFKA_MIRROR="https://s3-us-west-2.amazonaws.com/kafka-packages"
24+
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
2425
RUN mkdir -p "/opt/kafka-${KAFKA_VERSION}" \
2526
&& chmod a+rw "/opt/kafka-${KAFKA_VERSION}" \
2627
&& curl -s "$KAFKA_MIRROR/kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz" | tar xz --strip-components=1 -C "/opt/kafka-${KAFKA_VERSION}"
2728

2829
# older kafka versions depend upon jaxb-api being bundled with the JDK, but it
2930
# was removed from Java 11 so work around that by including it in the kafka
3031
# libs dir regardless
31-
RUN cd /tmp \
32-
&& curl -sLO "https://repo1.maven.org/maven2/javax/xml/bind/jaxb-api/2.3.0/jaxb-api-2.3.0.jar" \
32+
WORKDIR /tmp
33+
RUN curl -sLO "https://repo1.maven.org/maven2/javax/xml/bind/jaxb-api/2.3.0/jaxb-api-2.3.0.jar" \
3334
&& for DIR in /opt/kafka-*; do cp -v jaxb-api-2.3.0.jar $DIR/libs/ ; done \
3435
&& rm -f jaxb-api-2.3.0.jar
3536

@@ -41,4 +42,6 @@ RUN sed -e "s/JAVA_MAJOR_VERSION=.*/JAVA_MAJOR_VERSION=${JAVA_MAJOR_VERSION}/" -
4142

4243
COPY entrypoint.sh /
4344

45+
USER 65534:65534
46+
4447
ENTRYPOINT ["/entrypoint.sh"]

‎docker-compose.yml

+20-20
Original file line numberDiff line numberDiff line change
@@ -38,18 +38,18 @@ services:
3838
ZOO_4LW_COMMANDS_WHITELIST: 'mntr,conf,ruok'
3939
kafka-1:
4040
hostname: 'kafka-1'
41-
image: 'sarama/fv-kafka-${KAFKA_VERSION:-3.5.1}'
41+
image: 'sarama/fv-kafka-${KAFKA_VERSION:-3.6.0}'
4242
build:
4343
context: .
4444
dockerfile: Dockerfile.kafka
4545
args:
46-
KAFKA_VERSION: ${KAFKA_VERSION:-3.5.1}
46+
KAFKA_VERSION: ${KAFKA_VERSION:-3.6.0}
4747
SCALA_VERSION: ${SCALA_VERSION:-2.13}
4848
healthcheck:
4949
test:
5050
[
5151
'CMD',
52-
'/opt/kafka-${KAFKA_VERSION:-3.5.1}/bin/kafka-broker-api-versions.sh',
52+
'/opt/kafka-${KAFKA_VERSION:-3.6.0}/bin/kafka-broker-api-versions.sh',
5353
'--bootstrap-server',
5454
'kafka-1:9091',
5555
]
@@ -64,7 +64,7 @@ services:
6464
- toxiproxy
6565
restart: always
6666
environment:
67-
KAFKA_VERSION: ${KAFKA_VERSION:-3.5.1}
67+
KAFKA_VERSION: ${KAFKA_VERSION:-3.6.0}
6868
KAFKA_CFG_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181'
6969
KAFKA_CFG_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29091'
7070
KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-1:9091,LISTENER_LOCAL://localhost:29091'
@@ -83,18 +83,18 @@ services:
8383
KAFKA_JVM_PERFORMANCE_OPTS: "-XX:+IgnoreUnrecognizedVMOptions"
8484
kafka-2:
8585
hostname: 'kafka-2'
86-
image: 'sarama/fv-kafka-${KAFKA_VERSION:-3.5.1}'
86+
image: 'sarama/fv-kafka-${KAFKA_VERSION:-3.6.0}'
8787
build:
8888
context: .
8989
dockerfile: Dockerfile.kafka
9090
args:
91-
KAFKA_VERSION: ${KAFKA_VERSION:-3.5.1}
91+
KAFKA_VERSION: ${KAFKA_VERSION:-3.6.0}
9292
SCALA_VERSION: ${SCALA_VERSION:-2.13}
9393
healthcheck:
9494
test:
9595
[
9696
'CMD',
97-
'/opt/kafka-${KAFKA_VERSION:-3.5.1}/bin/kafka-broker-api-versions.sh',
97+
'/opt/kafka-${KAFKA_VERSION:-3.6.0}/bin/kafka-broker-api-versions.sh',
9898
'--bootstrap-server',
9999
'kafka-2:9091',
100100
]
@@ -109,7 +109,7 @@ services:
109109
- toxiproxy
110110
restart: always
111111
environment:
112-
KAFKA_VERSION: ${KAFKA_VERSION:-3.5.1}
112+
KAFKA_VERSION: ${KAFKA_VERSION:-3.6.0}
113113
KAFKA_CFG_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181'
114114
KAFKA_CFG_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29092'
115115
KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-2:9091,LISTENER_LOCAL://localhost:29092'
@@ -128,18 +128,18 @@ services:
128128
KAFKA_JVM_PERFORMANCE_OPTS: "-XX:+IgnoreUnrecognizedVMOptions"
129129
kafka-3:
130130
hostname: 'kafka-3'
131-
image: 'sarama/fv-kafka-${KAFKA_VERSION:-3.5.1}'
131+
image: 'sarama/fv-kafka-${KAFKA_VERSION:-3.6.0}'
132132
build:
133133
context: .
134134
dockerfile: Dockerfile.kafka
135135
args:
136-
KAFKA_VERSION: ${KAFKA_VERSION:-3.5.1}
136+
KAFKA_VERSION: ${KAFKA_VERSION:-3.6.0}
137137
SCALA_VERSION: ${SCALA_VERSION:-2.13}
138138
healthcheck:
139139
test:
140140
[
141141
'CMD',
142-
'/opt/kafka-${KAFKA_VERSION:-3.5.1}/bin/kafka-broker-api-versions.sh',
142+
'/opt/kafka-${KAFKA_VERSION:-3.6.0}/bin/kafka-broker-api-versions.sh',
143143
'--bootstrap-server',
144144
'kafka-3:9091',
145145
]
@@ -154,7 +154,7 @@ services:
154154
- toxiproxy
155155
restart: always
156156
environment:
157-
KAFKA_VERSION: ${KAFKA_VERSION:-3.5.1}
157+
KAFKA_VERSION: ${KAFKA_VERSION:-3.6.0}
158158
KAFKA_CFG_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181'
159159
KAFKA_CFG_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29093'
160160
KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-3:9091,LISTENER_LOCAL://localhost:29093'
@@ -173,18 +173,18 @@ services:
173173
KAFKA_JVM_PERFORMANCE_OPTS: "-XX:+IgnoreUnrecognizedVMOptions"
174174
kafka-4:
175175
hostname: 'kafka-4'
176-
image: 'sarama/fv-kafka-${KAFKA_VERSION:-3.5.1}'
176+
image: 'sarama/fv-kafka-${KAFKA_VERSION:-3.6.0}'
177177
build:
178178
context: .
179179
dockerfile: Dockerfile.kafka
180180
args:
181-
KAFKA_VERSION: ${KAFKA_VERSION:-3.5.1}
181+
KAFKA_VERSION: ${KAFKA_VERSION:-3.6.0}
182182
SCALA_VERSION: ${SCALA_VERSION:-2.13}
183183
healthcheck:
184184
test:
185185
[
186186
'CMD',
187-
'/opt/kafka-${KAFKA_VERSION:-3.5.1}/bin/kafka-broker-api-versions.sh',
187+
'/opt/kafka-${KAFKA_VERSION:-3.6.0}/bin/kafka-broker-api-versions.sh',
188188
'--bootstrap-server',
189189
'kafka-4:9091',
190190
]
@@ -199,7 +199,7 @@ services:
199199
- toxiproxy
200200
restart: always
201201
environment:
202-
KAFKA_VERSION: ${KAFKA_VERSION:-3.5.1}
202+
KAFKA_VERSION: ${KAFKA_VERSION:-3.6.0}
203203
KAFKA_CFG_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181'
204204
KAFKA_CFG_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29094'
205205
KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-4:9091,LISTENER_LOCAL://localhost:29094'
@@ -218,18 +218,18 @@ services:
218218
KAFKA_JVM_PERFORMANCE_OPTS: "-XX:+IgnoreUnrecognizedVMOptions"
219219
kafka-5:
220220
hostname: 'kafka-5'
221-
image: 'sarama/fv-kafka-${KAFKA_VERSION:-3.5.1}'
221+
image: 'sarama/fv-kafka-${KAFKA_VERSION:-3.6.0}'
222222
build:
223223
context: .
224224
dockerfile: Dockerfile.kafka
225225
args:
226-
KAFKA_VERSION: ${KAFKA_VERSION:-3.5.1}
226+
KAFKA_VERSION: ${KAFKA_VERSION:-3.6.0}
227227
SCALA_VERSION: ${SCALA_VERSION:-2.13}
228228
healthcheck:
229229
test:
230230
[
231231
'CMD',
232-
'/opt/kafka-${KAFKA_VERSION:-3.5.1}/bin/kafka-broker-api-versions.sh',
232+
'/opt/kafka-${KAFKA_VERSION:-3.6.0}/bin/kafka-broker-api-versions.sh',
233233
'--bootstrap-server',
234234
'kafka-5:9091',
235235
]
@@ -244,7 +244,7 @@ services:
244244
- toxiproxy
245245
restart: always
246246
environment:
247-
KAFKA_VERSION: ${KAFKA_VERSION:-3.5.1}
247+
KAFKA_VERSION: ${KAFKA_VERSION:-3.6.0}
248248
KAFKA_CFG_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181'
249249
KAFKA_CFG_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29095'
250250
KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-5:9091,LISTENER_LOCAL://localhost:29095'

‎entrypoint.sh

+11-8
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
11
#!/bin/bash
22

3-
KAFKA_VERSION="${KAFKA_VERSION:-3.5.1}"
3+
set -eu
4+
set -o pipefail
5+
6+
KAFKA_VERSION="${KAFKA_VERSION:-3.6.0}"
47
KAFKA_HOME="/opt/kafka-${KAFKA_VERSION}"
58

69
if [ ! -d "${KAFKA_HOME}" ]; then
@@ -10,19 +13,19 @@ fi
1013

1114
cd "${KAFKA_HOME}" || exit 1
1215

13-
# discard all empty/commented lines
14-
sed -e '/^#/d' -e '/^$/d' -i".orig" config/server.properties
16+
# discard all empty/commented lines from default config and copy to /tmp
17+
sed -e '/^#/d' -e '/^$/d' config/server.properties >/tmp/server.properties
1518

16-
echo "########################################################################" >>config/server.properties
19+
echo "########################################################################" >>/tmp/server.properties
1720

1821
# emulate kafka_configure_from_environment_variables from bitnami/bitnami-docker-kafka
1922
for var in "${!KAFKA_CFG_@}"; do
2023
key="$(echo "$var" | sed -e 's/^KAFKA_CFG_//g' -e 's/_/\./g' -e 's/.*/\L&/')"
21-
sed -e '/^'$key'/d' -i"" config/server.properties
24+
sed -e '/^'$key'/d' -i"" /tmp/server.properties
2225
value="${!var}"
23-
echo "$key=$value" >>config/server.properties
26+
echo "$key=$value" >>/tmp/server.properties
2427
done
2528

26-
sort config/server.properties
29+
sort /tmp/server.properties
2730

28-
exec bin/kafka-server-start.sh config/server.properties
31+
exec bin/kafka-server-start.sh /tmp/server.properties

‎utils.go

+4-2
Original file line numberDiff line numberDiff line change
@@ -198,6 +198,7 @@ var (
198198
V3_4_1_0 = newKafkaVersion(3, 4, 1, 0)
199199
V3_5_0_0 = newKafkaVersion(3, 5, 0, 0)
200200
V3_5_1_0 = newKafkaVersion(3, 5, 1, 0)
201+
V3_6_0_0 = newKafkaVersion(3, 6, 0, 0)
201202

202203
SupportedVersions = []KafkaVersion{
203204
V0_8_2_0,
@@ -258,9 +259,10 @@ var (
258259
V3_4_1_0,
259260
V3_5_0_0,
260261
V3_5_1_0,
262+
V3_6_0_0,
261263
}
262264
MinVersion = V0_8_2_0
263-
MaxVersion = V3_5_1_0
265+
MaxVersion = V3_6_0_0
264266
DefaultVersion = V2_1_0_0
265267

266268
// reduced set of protocol versions to matrix test
@@ -275,8 +277,8 @@ var (
275277
V2_6_2_0,
276278
V2_8_2_0,
277279
V3_1_2_0,
278-
V3_2_3_0,
279280
V3_3_2_0,
281+
V3_6_0_0,
280282
}
281283
)
282284

0 commit comments

Comments
 (0)