(ci) Run e2e tests as part of PR/CI checks #6
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: E2E Tests | |
| on: | |
| push: | |
| branches: ["master", "main"] | |
| pull_request: | |
| branches: ["master", "main"] | |
| workflow_dispatch: | |
| jobs: | |
| e2e-tests: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 15 | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v5 | |
| - name: Set up Go | |
| uses: actions/setup-go@v5 | |
| with: | |
| go-version-file: 'go.mod' | |
| - name: Start Kafka 4 with KRaft | |
| run: | | |
| docker run -d \ | |
| --name broker \ | |
| --network host \ | |
| -e KAFKA_NODE_ID=1 \ | |
| -e KAFKA_PROCESS_ROLES=broker,controller \ | |
| -e KAFKA_LISTENERS=PLAINTEXT://localhost:9092,CONTROLLER://localhost:9093 \ | |
| -e KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://localhost:9092 \ | |
| -e KAFKA_CONTROLLER_LISTENER_NAMES=CONTROLLER \ | |
| -e KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT \ | |
| -e KAFKA_CONTROLLER_QUORUM_VOTERS=1@localhost:9093 \ | |
| -e KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1 \ | |
| -e KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR=1 \ | |
| -e KAFKA_TRANSACTION_STATE_LOG_MIN_ISR=1 \ | |
| -e KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0 \ | |
| -e KAFKA_NUM_PARTITIONS=3 \ | |
| apache/kafka:4.1.0 | |
| # Wait for Kafka to be ready | |
| echo "Waiting for Kafka to start..." | |
| for i in {1..60}; do | |
| # Check if Kafka is listening on port 9092 | |
| if nc -z localhost 9092 2>/dev/null; then | |
| echo "Kafka port 9092 is open!" | |
| # Give it a couple more seconds to fully initialize | |
| sleep 3 | |
| # Try to get broker API versions | |
| if docker exec broker /opt/kafka/bin/kafka-broker-api-versions.sh --bootstrap-server localhost:9092 > /dev/null 2>&1; then | |
| echo "Kafka is ready and responding to requests!" | |
| break | |
| fi | |
| fi | |
| if [ $i -eq 60 ]; then | |
| echo "Kafka failed to start within 60 seconds" | |
| echo "=== Kafka logs ===" | |
| docker logs broker | |
| exit 1 | |
| fi | |
| echo "Attempt $i/60: Kafka not ready yet, waiting..." | |
| sleep 1 | |
| done | |
| # Verify Kafka is running | |
| echo "Verifying Kafka cluster..." | |
| docker exec broker /opt/kafka/bin/kafka-broker-api-versions.sh --bootstrap-server localhost:9092 | |
| echo "Kafka cluster is up and running!" | |
| - name: Create KMinion configuration | |
| run: | | |
| cat > kminion-config.yaml << 'EOF' | |
| kafka: | |
| brokers: | |
| - localhost:9092 | |
| clientId: kminion-e2e-test | |
| minion: | |
| endToEnd: | |
| enabled: true | |
| probeInterval: 500ms | |
| topicManagement: | |
| enabled: true | |
| name: kminion-end-to-end | |
| replicationFactor: 1 | |
| partitionsPerBroker: 1 | |
| reconciliationInterval: 10s | |
| producer: | |
| ackSla: 5s | |
| requiredAcks: all | |
| consumer: | |
| groupIdPrefix: kminion-end-to-end | |
| deleteStaleConsumerGroups: false | |
| roundtripSla: 10s | |
| commitSla: 5s | |
| exporter: | |
| namespace: kminion | |
| port: 8080 | |
| logger: | |
| level: info | |
| EOF | |
| - name: Build KMinion | |
| run: | | |
| go build -o kminion . | |
| - name: Start KMinion with E2E tests | |
| run: | | |
| export CONFIG_FILEPATH=kminion-config.yaml | |
| ./kminion > kminion.log 2>&1 & | |
| KMINION_PID=$! | |
| echo "KMINION_PID=$KMINION_PID" >> $GITHUB_ENV | |
| # Wait for KMinion to start | |
| echo "Waiting for KMinion to start..." | |
| for i in {1..30}; do | |
| if curl -s http://localhost:8080/metrics > /dev/null 2>&1; then | |
| echo "KMinion is ready!" | |
| break | |
| fi | |
| if [ $i -eq 30 ]; then | |
| echo "KMinion failed to start within 30 seconds" | |
| cat kminion.log | |
| exit 1 | |
| fi | |
| echo "Attempt $i/30: KMinion not ready yet, waiting..." | |
| sleep 1 | |
| done | |
| - name: Wait for E2E tests to run | |
| run: | | |
| echo "Waiting for E2E tests to produce metrics..." | |
| sleep 30 | |
| - name: Validate E2E metrics | |
| run: | | |
| echo "Fetching metrics from KMinion..." | |
| METRICS=$(curl -s http://localhost:8080/metrics) | |
| echo "=== KMinion Metrics Output ===" | |
| echo "$METRICS" | grep -E "kminion_end_to_end_" || true | |
| echo "==============================" | |
| # Check for essential E2E metrics | |
| REQUIRED_METRICS=( | |
| "kminion_end_to_end_messages_produced_total" | |
| "kminion_end_to_end_messages_received_total" | |
| "kminion_end_to_end_produce_latency_seconds" | |
| "kminion_end_to_end_roundtrip_latency_seconds" | |
| "kminion_end_to_end_offset_commit_latency_seconds" | |
| ) | |
| MISSING_METRICS=() | |
| for metric in "${REQUIRED_METRICS[@]}"; do | |
| if ! echo "$METRICS" | grep -q "^${metric}"; then | |
| MISSING_METRICS+=("$metric") | |
| fi | |
| done | |
| if [ ${#MISSING_METRICS[@]} -ne 0 ]; then | |
| echo "ERROR: Missing required E2E metrics:" | |
| printf '%s\n' "${MISSING_METRICS[@]}" | |
| echo "" | |
| echo "=== Full KMinion logs ===" | |
| cat kminion.log | |
| exit 1 | |
| fi | |
| # Verify messages were produced and received | |
| PRODUCED=$(echo "$METRICS" | grep "^kminion_end_to_end_messages_produced_total" | awk '{print $2}') | |
| RECEIVED=$(echo "$METRICS" | grep "^kminion_end_to_end_messages_received_total" | awk '{print $2}') | |
| echo "Messages produced: $PRODUCED" | |
| echo "Messages received: $RECEIVED" | |
| if [ -z "$PRODUCED" ] || [ "$PRODUCED" = "0" ]; then | |
| echo "ERROR: No messages were produced" | |
| cat kminion.log | |
| exit 1 | |
| fi | |
| if [ -z "$RECEIVED" ] || [ "$RECEIVED" = "0" ]; then | |
| echo "ERROR: No messages were received" | |
| cat kminion.log | |
| exit 1 | |
| fi | |
| echo "✅ E2E tests passed successfully!" | |
| echo " - Messages produced: $PRODUCED" | |
| echo " - Messages received: $RECEIVED" | |
| - name: Validate built-in KMinion metrics | |
| run: | | |
| echo "Validating built-in KMinion metrics..." | |
| METRICS=$(curl -s http://localhost:8080/metrics) | |
| # Core exporter metrics | |
| CORE_METRICS=( | |
| "kminion_exporter_up" | |
| "kminion_exporter_offset_consumer_records_consumed_total" | |
| ) | |
| # Kafka cluster metrics | |
| KAFKA_METRICS=( | |
| "kminion_kafka_cluster_info" | |
| "kminion_kafka_broker_info" | |
| ) | |
| # Topic metrics | |
| TOPIC_METRICS=( | |
| "kminion_kafka_topic_info" | |
| "kminion_kafka_topic_partition_count" | |
| "kminion_kafka_topic_partition_high_water_mark" | |
| "kminion_kafka_topic_high_water_mark_sum" | |
| ) | |
| MISSING_METRICS=() | |
| # Check core metrics | |
| for metric in "${CORE_METRICS[@]}"; do | |
| if ! echo "$METRICS" | grep -q "^${metric}"; then | |
| MISSING_METRICS+=("$metric") | |
| fi | |
| done | |
| # Check Kafka metrics | |
| for metric in "${KAFKA_METRICS[@]}"; do | |
| if ! echo "$METRICS" | grep -q "^${metric}"; then | |
| MISSING_METRICS+=("$metric") | |
| fi | |
| done | |
| # Check topic metrics | |
| for metric in "${TOPIC_METRICS[@]}"; do | |
| if ! echo "$METRICS" | grep -q "^${metric}"; then | |
| MISSING_METRICS+=("$metric") | |
| fi | |
| done | |
| if [ ${#MISSING_METRICS[@]} -ne 0 ]; then | |
| echo "ERROR: Missing required built-in metrics:" | |
| printf '%s\n' "${MISSING_METRICS[@]}" | |
| echo "" | |
| echo "=== Available metrics ===" | |
| echo "$METRICS" | grep "^kminion_" | head -20 | |
| exit 1 | |
| fi | |
| # Validate specific metric values | |
| EXPORTER_UP=$(echo "$METRICS" | grep "^kminion_exporter_up" | awk '{print $2}') | |
| if [ "$EXPORTER_UP" != "1" ]; then | |
| echo "ERROR: kminion_exporter_up should be 1, got: $EXPORTER_UP" | |
| exit 1 | |
| fi | |
| # Check that cluster info has broker_count label | |
| if ! echo "$METRICS" | grep "kminion_kafka_cluster_info" | grep -q "broker_count"; then | |
| echo "ERROR: kminion_kafka_cluster_info missing broker_count label" | |
| exit 1 | |
| fi | |
| # Check that we have broker info for at least one broker | |
| BROKER_COUNT=$(echo "$METRICS" | grep "^kminion_kafka_broker_info" | wc -l) | |
| if [ "$BROKER_COUNT" -lt 1 ]; then | |
| echo "ERROR: No broker info metrics found" | |
| exit 1 | |
| fi | |
| echo "✅ Built-in KMinion metrics validation passed!" | |
| echo " - Exporter up: $EXPORTER_UP" | |
| echo " - Brokers detected: $BROKER_COUNT" | |
| - name: Show logs on failure | |
| if: failure() | |
| run: | | |
| echo "=== Kafka logs ===" | |
| docker logs broker || echo "No Kafka logs found" | |
| echo "" | |
| echo "=== KMinion logs ===" | |
| cat kminion.log || echo "No KMinion logs found" | |
| - name: Cleanup | |
| if: always() | |
| run: | | |
| # Stop KMinion | |
| if [ ! -z "$KMINION_PID" ]; then | |
| kill $KMINION_PID || true | |
| fi | |
| # Stop and remove Kafka container | |
| docker stop broker || true | |
| docker rm broker || true | |
| # Wait a bit for graceful shutdown | |
| sleep 2 | |