Skip to content

Commit 829e8c0

Browse files
author
amuraru
committed
(ci) Run e2e tests as part of PR/CI checks
1 parent be6d416 commit 829e8c0

File tree

1 file changed

+248
-0
lines changed

1 file changed

+248
-0
lines changed

‎.github/workflows/e2e-tests.yaml‎

Lines changed: 248 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,248 @@
1+
name: E2E Tests
2+
3+
on:
4+
push:
5+
branches: ["master", "main"]
6+
pull_request:
7+
branches: ["master", "main"]
8+
workflow_dispatch:
9+
10+
jobs:
11+
e2e-tests:
12+
runs-on: ubuntu-latest
13+
timeout-minutes: 15
14+
15+
steps:
16+
- name: Checkout code
17+
uses: actions/checkout@v5
18+
19+
- name: Set up Go
20+
uses: actions/setup-go@v5
21+
with:
22+
go-version-file: 'go.mod'
23+
24+
- name: Create Kafka KRaft configuration
25+
run: |
26+
mkdir -p kafka-config
27+
cat > kafka-config/server.properties << 'EOF'
28+
# KRaft mode configuration
29+
process.roles=broker,controller
30+
node.id=1
31+
controller.quorum.voters=1@localhost:9093
32+
33+
# Listeners
34+
listeners=PLAINTEXT://0.0.0.0:9092,CONTROLLER://0.0.0.0:9093
35+
advertised.listeners=PLAINTEXT://localhost:9092
36+
listener.security.protocol.map=PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT
37+
controller.listener.names=CONTROLLER
38+
inter.broker.listener.name=PLAINTEXT
39+
40+
# Log directories
41+
log.dirs=/tmp/kraft-combined-logs
42+
43+
# Cluster ID will be generated
44+
# cluster.id will be set dynamically
45+
46+
# Topic defaults
47+
offsets.topic.replication.factor=1
48+
transaction.state.log.replication.factor=1
49+
transaction.state.log.min.isr=1
50+
default.replication.factor=1
51+
min.insync.replicas=1
52+
53+
# Performance tuning for CI
54+
num.network.threads=3
55+
num.io.threads=8
56+
socket.send.buffer.bytes=102400
57+
socket.receive.buffer.bytes=102400
58+
socket.request.max.bytes=104857600
59+
EOF
60+
61+
- name: Start Kafka 4.0 with KRaft
62+
run: |
63+
# Download and extract Kafka 4.0
64+
wget -q https://archive.apache.org/dist/kafka/4.0.0/kafka_2.13-4.0.0.tgz
65+
tar -xzf kafka_2.13-4.0.0.tgz
66+
67+
# Generate cluster ID
68+
CLUSTER_ID=$(kafka_2.13-4.0.0/bin/kafka-storage.sh random-uuid)
69+
echo "Generated Cluster ID: $CLUSTER_ID"
70+
71+
# Format storage
72+
kafka_2.13-4.0.0/bin/kafka-storage.sh format \
73+
-t $CLUSTER_ID \
74+
-c kafka-config/server.properties
75+
76+
# Start Kafka in background
77+
kafka_2.13-4.0.0/bin/kafka-server-start.sh kafka-config/server.properties > kafka.log 2>&1 &
78+
KAFKA_PID=$!
79+
echo "KAFKA_PID=$KAFKA_PID" >> $GITHUB_ENV
80+
81+
# Wait for Kafka to be ready
82+
echo "Waiting for Kafka to start..."
83+
for i in {1..30}; do
84+
if kafka_2.13-4.0.0/bin/kafka-broker-api-versions.sh --bootstrap-server localhost:9092 > /dev/null 2>&1; then
85+
echo "Kafka is ready!"
86+
break
87+
fi
88+
if [ $i -eq 30 ]; then
89+
echo "Kafka failed to start within 30 seconds"
90+
cat kafka.log
91+
exit 1
92+
fi
93+
echo "Attempt $i/30: Kafka not ready yet, waiting..."
94+
sleep 1
95+
done
96+
97+
# Verify Kafka is running
98+
kafka_2.13-4.0.0/bin/kafka-broker-api-versions.sh --bootstrap-server localhost:9092
99+
100+
- name: Create KMinion configuration
101+
run: |
102+
cat > kminion-config.yaml << 'EOF'
103+
kafka:
104+
brokers:
105+
- localhost:9092
106+
clientId: kminion-e2e-test
107+
108+
minion:
109+
endToEnd:
110+
enabled: true
111+
probeInterval: 500ms
112+
topicManagement:
113+
enabled: true
114+
name: kminion-end-to-end
115+
replicationFactor: 1
116+
partitionsPerBroker: 1
117+
reconciliationInterval: 10s
118+
producer:
119+
ackSla: 5s
120+
requiredAcks: all
121+
consumer:
122+
groupIdPrefix: kminion-end-to-end
123+
deleteStaleConsumerGroups: false
124+
roundtripSla: 10s
125+
commitSla: 5s
126+
127+
exporter:
128+
namespace: kminion
129+
port: 8080
130+
131+
logger:
132+
level: info
133+
EOF
134+
135+
- name: Build KMinion
136+
run: |
137+
go build -o kminion .
138+
139+
- name: Start KMinion with E2E tests
140+
run: |
141+
export CONFIG_FILEPATH=kminion-config.yaml
142+
./kminion > kminion.log 2>&1 &
143+
KMINION_PID=$!
144+
echo "KMINION_PID=$KMINION_PID" >> $GITHUB_ENV
145+
146+
# Wait for KMinion to start
147+
echo "Waiting for KMinion to start..."
148+
for i in {1..30}; do
149+
if curl -s http://localhost:8080/metrics > /dev/null 2>&1; then
150+
echo "KMinion is ready!"
151+
break
152+
fi
153+
if [ $i -eq 30 ]; then
154+
echo "KMinion failed to start within 30 seconds"
155+
cat kminion.log
156+
exit 1
157+
fi
158+
echo "Attempt $i/30: KMinion not ready yet, waiting..."
159+
sleep 1
160+
done
161+
162+
- name: Wait for E2E tests to run
163+
run: |
164+
echo "Waiting for E2E tests to produce metrics..."
165+
sleep 30
166+
167+
- name: Validate E2E metrics
168+
run: |
169+
echo "Fetching metrics from KMinion..."
170+
METRICS=$(curl -s http://localhost:8080/metrics)
171+
172+
echo "=== KMinion Metrics Output ==="
173+
echo "$METRICS" | grep -E "kminion_end_to_end_" || true
174+
echo "=============================="
175+
176+
# Check for essential E2E metrics
177+
REQUIRED_METRICS=(
178+
"kminion_end_to_end_messages_produced_total"
179+
"kminion_end_to_end_messages_received_total"
180+
"kminion_end_to_end_produce_latency_seconds"
181+
"kminion_end_to_end_roundtrip_latency_seconds"
182+
"kminion_end_to_end_offset_commit_latency_seconds"
183+
)
184+
185+
MISSING_METRICS=()
186+
for metric in "${REQUIRED_METRICS[@]}"; do
187+
if ! echo "$METRICS" | grep -q "^${metric}"; then
188+
MISSING_METRICS+=("$metric")
189+
fi
190+
done
191+
192+
if [ ${#MISSING_METRICS[@]} -ne 0 ]; then
193+
echo "ERROR: Missing required E2E metrics:"
194+
printf '%s\n' "${MISSING_METRICS[@]}"
195+
echo ""
196+
echo "=== Full KMinion logs ==="
197+
cat kminion.log
198+
exit 1
199+
fi
200+
201+
# Verify messages were produced and received
202+
PRODUCED=$(echo "$METRICS" | grep "^kminion_end_to_end_messages_produced_total" | awk '{print $2}')
203+
RECEIVED=$(echo "$METRICS" | grep "^kminion_end_to_end_messages_received_total" | awk '{print $2}')
204+
205+
echo "Messages produced: $PRODUCED"
206+
echo "Messages received: $RECEIVED"
207+
208+
if [ -z "$PRODUCED" ] || [ "$PRODUCED" = "0" ]; then
209+
echo "ERROR: No messages were produced"
210+
cat kminion.log
211+
exit 1
212+
fi
213+
214+
if [ -z "$RECEIVED" ] || [ "$RECEIVED" = "0" ]; then
215+
echo "ERROR: No messages were received"
216+
cat kminion.log
217+
exit 1
218+
fi
219+
220+
echo "✅ E2E tests passed successfully!"
221+
echo " - Messages produced: $PRODUCED"
222+
echo " - Messages received: $RECEIVED"
223+
224+
- name: Show logs on failure
225+
if: failure()
226+
run: |
227+
echo "=== Kafka logs ==="
228+
cat kafka.log || echo "No Kafka logs found"
229+
echo ""
230+
echo "=== KMinion logs ==="
231+
cat kminion.log || echo "No KMinion logs found"
232+
233+
- name: Cleanup
234+
if: always()
235+
run: |
236+
# Stop KMinion
237+
if [ ! -z "$KMINION_PID" ]; then
238+
kill $KMINION_PID || true
239+
fi
240+
241+
# Stop Kafka
242+
if [ ! -z "$KAFKA_PID" ]; then
243+
kill $KAFKA_PID || true
244+
fi
245+
246+
# Wait a bit for graceful shutdown
247+
sleep 2
248+

0 commit comments

Comments
 (0)