docker kafka

Author: amuraru

.github/workflows/e2e-tests.yaml

@@ -21,81 +21,53 @@ jobs:
         with:
           go-version-file: 'go.mod'
 
-      - name: Create Kafka KRaft configuration
-        run: |
-          mkdir -p kafka-config
-          cat > kafka-config/server.properties << 'EOF'
-          # KRaft mode configuration
-          process.roles=broker,controller
-          node.id=1
-          controller.quorum.voters=1@localhost:9093
-          
-          # Listeners
-          listeners=PLAINTEXT://0.0.0.0:9092,CONTROLLER://0.0.0.0:9093
-          advertised.listeners=PLAINTEXT://localhost:9092
-          listener.security.protocol.map=PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT
-          controller.listener.names=CONTROLLER
-          inter.broker.listener.name=PLAINTEXT
-          
-          # Log directories
-          log.dirs=/tmp/kraft-combined-logs
-          
-          # Cluster ID will be generated
-          # cluster.id will be set dynamically
-          
-          # Topic defaults
-          offsets.topic.replication.factor=1
-          transaction.state.log.replication.factor=1
-          transaction.state.log.min.isr=1
-          default.replication.factor=1
-          min.insync.replicas=1
-          
-          # Performance tuning for CI
-          num.network.threads=3
-          num.io.threads=8
-          socket.send.buffer.bytes=102400
-          socket.receive.buffer.bytes=102400
-          socket.request.max.bytes=104857600
-          EOF
-
       - name: Start Kafka 4.0 with KRaft
         run: |
-          # Download and extract Kafka 4.0
-          wget -q https://archive.apache.org/dist/kafka/4.0.0/kafka_2.13-4.0.0.tgz
-          tar -xzf kafka_2.13-4.0.0.tgz
-          
-          # Generate cluster ID
-          CLUSTER_ID=$(kafka_2.13-4.0.0/bin/kafka-storage.sh random-uuid)
-          echo "Generated Cluster ID: $CLUSTER_ID"
-          
-          # Format storage
-          kafka_2.13-4.0.0/bin/kafka-storage.sh format \
-            -t $CLUSTER_ID \
-            -c kafka-config/server.properties
-          
-          # Start Kafka in background
-          kafka_2.13-4.0.0/bin/kafka-server-start.sh kafka-config/server.properties > kafka.log 2>&1 &
-          KAFKA_PID=$!
-          echo "KAFKA_PID=$KAFKA_PID" >> $GITHUB_ENV
-          
+          docker run -d \
+            --name broker \
+            --network host \
+            -e KAFKA_NODE_ID=1 \
+            -e KAFKA_PROCESS_ROLES=broker,controller \
+            -e KAFKA_LISTENERS=PLAINTEXT://localhost:9092,CONTROLLER://localhost:9093 \
+            -e KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://localhost:9092 \
+            -e KAFKA_CONTROLLER_LISTENER_NAMES=CONTROLLER \
+            -e KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT \
+            -e KAFKA_CONTROLLER_QUORUM_VOTERS=1@localhost:9093 \
+            -e KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1 \
+            -e KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR=1 \
+            -e KAFKA_TRANSACTION_STATE_LOG_MIN_ISR=1 \
+            -e KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0 \
+            -e KAFKA_NUM_PARTITIONS=3 \
+            apache/kafka:4.0.0
+
           # Wait for Kafka to be ready
           echo "Waiting for Kafka to start..."
-          for i in {1..30}; do
-            if kafka_2.13-4.0.0/bin/kafka-broker-api-versions.sh --bootstrap-server localhost:9092 > /dev/null 2>&1; then
-              echo "Kafka is ready!"
-              break
+          for i in {1..60}; do
+            # Check if Kafka is listening on port 9092
+            if nc -z localhost 9092 2>/dev/null; then
+              echo "Kafka port 9092 is open!"
+              # Give it a couple more seconds to fully initialize
+              sleep 3
+              # Try to get broker API versions
+              if docker exec broker /opt/kafka/bin/kafka-broker-api-versions.sh --bootstrap-server localhost:9092 > /dev/null 2>&1; then
+                echo "Kafka is ready and responding to requests!"
+                break
+              fi
             fi
-            if [ $i -eq 30 ]; then
-              echo "Kafka failed to start within 30 seconds"
-              cat kafka.log
+            if [ $i -eq 60 ]; then
+              echo "Kafka failed to start within 60 seconds"
+              echo "=== Kafka logs ==="
+              docker logs broker
               exit 1
             fi
-            echo "Attempt $i/30: Kafka not ready yet, waiting..."
+            echo "Attempt $i/60: Kafka not ready yet, waiting..."
             sleep 1
           done
-          
+
           # Verify Kafka is running
-          kafka_2.13-4.0.0/bin/kafka-broker-api-versions.sh --bootstrap-server localhost:9092
+          echo "Verifying Kafka cluster..."
+          docker exec broker /opt/kafka/bin/kafka-broker-api-versions.sh --bootstrap-server localhost:9092
+          echo "Kafka cluster is up and running!"
 
       - name: Create KMinion configuration
         run: |
@@ -225,7 +197,7 @@ jobs:
         if: failure()
         run: |
           echo "=== Kafka logs ==="
-          cat kafka.log || echo "No Kafka logs found"
+          docker logs broker || echo "No Kafka logs found"
           echo ""
           echo "=== KMinion logs ==="
           cat kminion.log || echo "No KMinion logs found"
@@ -237,12 +209,11 @@ jobs:
           if [ ! -z "$KMINION_PID" ]; then
             kill $KMINION_PID || true
           fi
-          
-          # Stop Kafka
-          if [ ! -z "$KAFKA_PID" ]; then
-            kill $KAFKA_PID || true
-          fi
-          
+
+          # Stop and remove Kafka container
+          docker stop broker || true
+          docker rm broker || true
+
           # Wait a bit for graceful shutdown
           sleep 2