Skip to content

Commit 03f1a23

Browse files
Dhinakaran-2kDhinakaran-2k
committed
Working MM2 truncation + reset hardening with end-to-end test suite
1 parent 388939e commit 03f1a23

12 files changed

Lines changed: 839 additions & 473 deletions

File tree

‎.gitignore‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
# Kafka fork (kept as a separate repo / submitted as fork+PR)
2+
kafka-fork/
3+
4+
# Build artifacts
5+
**/target/
6+
**/build/
7+
**/.gradle/
8+
*.class
9+
*.jar
10+
11+
# IDE
12+
.idea/
13+
.vscode/
14+
*.iml
15+
16+
# OS
17+
.DS_Store
18+
Thumbs.db
19+
20+
# Project planning / scratch (not for submission)
21+
PLAN.md
22+
NOTES.md
23+
*.local.md
24+
25+
# Producer maven module
26+
producer/target/
27+
28+
# Mac/Win files
29+
.DS_Store

‎Dockerfile.mm2‎

Lines changed: 39 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,41 @@
1-
FROM apache/kafka:4.0.0
1+
# Enhanced MirrorMaker 2: apache/kafka:4.0.0 with a patched MirrorSourceTask.
2+
#
3+
# Rather than rebuilding all of Kafka (~30+ min), we recompile only the modified
4+
# source file against the jars from the official image, then drop the new .class
5+
# files into the existing connect-mirror jar. Same package, same classloader,
6+
# so package-private references resolve at runtime.
27

3-
# Copy your freshly compiled custom JAR engine modifications directly from the sibling build paths
4-
COPY connect/mirror/build/libs/connect-mirror-*.jar /opt/kafka/libs/
5-
COPY connect/mirror-client/build/libs/connect-mirror-client-*.jar /opt/kafka/libs/
8+
ARG KAFKA_IMAGE=apache/kafka:4.0.0
69

7-
# Create a placeholder directory to ensure smooth volume mount overrides
8-
RUN mkdir -p /opt/kafka/config/
10+
FROM ${KAFKA_IMAGE} AS kafka-base
11+
12+
FROM eclipse-temurin:17-jdk AS builder
13+
WORKDIR /work
14+
15+
# Pull the runtime libs so we can compile against them
16+
COPY --from=kafka-base /opt/kafka/libs /kafka-libs
17+
18+
# Only the file we changed (mirrored from the apache/kafka fork at kafka-fork/connect/mirror/...)
19+
COPY mm2-patch/org/apache/kafka/connect/mirror/MirrorSourceTask.java \
20+
/work/src/org/apache/kafka/connect/mirror/MirrorSourceTask.java
21+
22+
RUN mkdir -p /work/classes && \
23+
javac -source 17 -target 17 -d /work/classes -cp "/kafka-libs/*" \
24+
/work/src/org/apache/kafka/connect/mirror/MirrorSourceTask.java
25+
26+
# Find the existing connect-mirror jar (not the -client variant) and overlay our class files
27+
RUN SRC_JAR=$(ls /kafka-libs/connect-mirror-[0-9]*.jar | head -1) && \
28+
echo "patching: $SRC_JAR" && \
29+
cp "$SRC_JAR" /work/connect-mirror.jar && \
30+
cd /work/classes && \
31+
jar uf /work/connect-mirror.jar org/apache/kafka/connect/mirror/MirrorSourceTask.class && \
32+
(ls org/apache/kafka/connect/mirror/MirrorSourceTask\$*.class 2>/dev/null | \
33+
xargs -r jar uf /work/connect-mirror.jar || true)
34+
35+
FROM ${KAFKA_IMAGE}
36+
USER root
37+
COPY --from=builder /work/connect-mirror.jar /tmp/connect-mirror-patched.jar
38+
RUN ORIG=$(ls /opt/kafka/libs/connect-mirror-[0-9]*.jar | head -1) && \
39+
cp /tmp/connect-mirror-patched.jar "$ORIG" && \
40+
rm /tmp/connect-mirror-patched.jar
41+
USER appuser

‎README.md‎

Lines changed: 164 additions & 155 deletions
Large diffs are not rendered by default.

‎docker-compose.yml‎

Lines changed: 41 additions & 77 deletions
Original file line numberDiff line numberDiff line change
@@ -1,114 +1,78 @@
1-
version: "3.9"
2-
31
services:
42
primary:
53
image: apache/kafka:4.0.0
4+
container_name: primary
5+
hostname: primary
6+
ports:
7+
- "9092:9092"
68
environment:
79
KAFKA_NODE_ID: 1
8-
KAFKA_PROCESS_ROLES: "broker,controller"
9-
10-
KAFKA_CONTROLLER_QUORUM_VOTERS: "1@primary:9093"
11-
12-
KAFKA_LISTENERS: "PLAINTEXT://0.0.0.0:9092,CONTROLLER://0.0.0.0:9093"
13-
14-
KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://primary:9092"
15-
16-
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: "PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT"
17-
18-
KAFKA_INTER_BROKER_LISTENER_NAME: "PLAINTEXT"
19-
20-
KAFKA_CONTROLLER_LISTENER_NAMES: "CONTROLLER"
21-
22-
KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true"
23-
10+
KAFKA_PROCESS_ROLES: broker,controller
11+
KAFKA_CONTROLLER_QUORUM_VOTERS: 1@primary:9093
12+
KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092,CONTROLLER://0.0.0.0:9093
13+
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://primary:9092
14+
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT
15+
KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
16+
KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
17+
KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
2418
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
2519
KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
2620
KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
2721
KAFKA_DEFAULT_REPLICATION_FACTOR: 1
2822
KAFKA_MIN_INSYNC_REPLICAS: 1
29-
3023
KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
31-
24+
# Aggressive retention settings so the truncation scenario is reproducible
3225
KAFKA_LOG_RETENTION_CHECK_INTERVAL_MS: 1000
3326
KAFKA_LOG_SEGMENT_DELETE_DELAY_MS: 1000
34-
KAFKA_LOG_SEGMENT_MS: 1000
35-
KAFKA_LOG_SEGMENT_BYTES: 1048576
36-
37-
ports:
38-
- "9092:9092"
3927

4028
standby:
4129
image: apache/kafka:4.0.0
30+
container_name: standby
31+
hostname: standby
32+
ports:
33+
- "9094:9094"
4234
environment:
4335
KAFKA_NODE_ID: 2
44-
KAFKA_PROCESS_ROLES: "broker,controller"
45-
46-
KAFKA_CONTROLLER_QUORUM_VOTERS: "2@standby:9093"
47-
48-
KAFKA_LISTENERS: "PLAINTEXT://0.0.0.0:9094,CONTROLLER://0.0.0.0:9093"
49-
50-
KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://standby:9094"
51-
52-
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: "PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT"
53-
54-
KAFKA_INTER_BROKER_LISTENER_NAME: "PLAINTEXT"
55-
56-
KAFKA_CONTROLLER_LISTENER_NAMES: "CONTROLLER"
57-
36+
KAFKA_PROCESS_ROLES: broker,controller
37+
KAFKA_CONTROLLER_QUORUM_VOTERS: 2@standby:9093
38+
KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9094,CONTROLLER://0.0.0.0:9093
39+
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://standby:9094
40+
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT
41+
KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
42+
KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
5843
KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true"
59-
6044
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
6145
KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
6246
KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
6347
KAFKA_DEFAULT_REPLICATION_FACTOR: 1
6448
KAFKA_MIN_INSYNC_REPLICAS: 1
65-
6649
KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
6750

68-
KAFKA_LOG_RETENTION_CHECK_INTERVAL_MS: 1000
69-
KAFKA_LOG_SEGMENT_DELETE_DELAY_MS: 1000
70-
KAFKA_LOG_SEGMENT_MS: 1000
71-
KAFKA_LOG_SEGMENT_BYTES: 1048576
72-
73-
ports:
74-
- "9094:9094"
75-
76-
producer:
77-
build: ./producer
78-
depends_on:
79-
- primary
80-
81-
environment:
82-
KAFKA_BOOTSTRAP_SERVERS: "primary:9092"
83-
84-
command:
85-
[
86-
"--count",
87-
"1000",
88-
"--bootstrap-servers",
89-
"primary:9092",
90-
"--topic",
91-
"commit-log"
92-
]
93-
9451
mirror-maker:
9552
build:
96-
context: ../kafka
97-
dockerfile: ../challenge-env/Dockerfile.mm2
98-
53+
context: .
54+
dockerfile: Dockerfile.mm2
55+
image: kafka-mm2-enhanced:local
56+
container_name: mirror-maker
9957
depends_on:
10058
- primary
10159
- standby
102-
10360
volumes:
10461
- ./mm2.properties:/opt/kafka/config/mm2.properties:ro
105-
10662
entrypoint:
107-
[
108-
"/opt/kafka/bin/connect-mirror-maker.sh",
109-
"/opt/kafka/config/mm2.properties"
110-
]
63+
- /opt/kafka/bin/connect-mirror-maker.sh
64+
- /opt/kafka/config/mm2.properties
65+
66+
producer:
67+
build:
68+
context: ./producer
69+
image: commit-log-producer:local
70+
container_name: producer
71+
depends_on:
72+
- primary
73+
# Overridden by run_challenge.sh as needed
74+
command: ["--count", "1000", "--bootstrap-servers", "primary:9092", "--topic", "commit-log"]
11175

11276
networks:
11377
default:
114-
name: kafka-challenge-net
78+
name: kafka-mm2-net

‎mm2-patch/README.md‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
# MM2 source patch
2+
3+
This directory holds the single Java source file that diverges from Apache
4+
Kafka 4.0.0 — `org/apache/kafka/connect/mirror/MirrorSourceTask.java`.
5+
6+
The same file lives in the Kafka fork at
7+
`connect/mirror/src/main/java/org/apache/kafka/connect/mirror/MirrorSourceTask.java`;
8+
the two must stay in sync. `Dockerfile.mm2` reads from this directory so the
9+
challenge repo can build the enhanced image without checking in the whole
10+
Kafka tree.
11+
12+
See `../README.md` for what was changed and why.

0 commit comments

Comments
 (0)