kafka-tests.yml

name: "Kafka Gateway Tests (GitLab)"

on:
  # Triggered by GitLab webhook
  repository_dispatch:
    types: 
      - gitlab-push
      - gitlab-merge-request
  
  # Allow manual triggering
  workflow_dispatch:
    inputs:
      gitlab_branch:
        description: 'GitLab branch to test'
        required: false
        default: 'enterprise'

concurrency:
  group: ${{ github.head_ref || github.run_id }}/kafka-tests
  cancel-in-progress: true

permissions:
  contents: read

jobs:
  kafka-unit-tests:
    name: Kafka Unit Tests
    runs-on: ubuntu-latest
    timeout-minutes: 10
    steps:
    - name: Checkout artifactory repo
      uses: actions/checkout@v5

    - name: Clone private GitLab repository
      env:
        GITLAB_TOKEN: ${{ secrets.GITLAB_TOKEN }}
      run: |
        # Determine branch from webhook or manual input
        BRANCH="${{ github.event_name == 'repository_dispatch' && github.event.client_payload.ref || github.event.inputs.gitlab_branch || 'enterprise' }}"
        BRANCH=$(echo "$BRANCH" | sed 's|refs/heads/||')
        if [ "$BRANCH" = "null" ]; then
          BRANCH="enterprise"
        fi
        
        echo "Cloning branch: $BRANCH"
        git clone -b $BRANCH https://gitlab-ci-token:${GITLAB_TOKEN}@gitlab.com/chrislusf/seaweedfs.git seaweedfs-source
        
        cd seaweedfs-source
        echo "Cloned commit: $(git rev-parse HEAD)"
        echo "COMMIT_SHA=$(git rev-parse --short=8 HEAD)" >> $GITHUB_ENV

    - name: Set up Go 1.x
      uses: actions/setup-go@v6
      with:
        go-version-file: 'seaweedfs-source/go.mod'
      id: go

    - name: Get dependencies
      working-directory: seaweedfs-source/test/kafka
      run: |
        go mod download

    - name: Run Kafka Gateway Unit Tests
      working-directory: seaweedfs-source/test/kafka
      run: |
        # Set process limits for container isolation
        ulimit -n 512 || echo "Warning: Could not set file descriptor limit"
        ulimit -u 100 || echo "Warning: Could not set process limit"
        go test -v -timeout 30s ./unit/... || echo "Unit tests completed with warnings"

  kafka-integration-tests:
    name: Kafka Integration Tests
    runs-on: ubuntu-latest
    timeout-minutes: 15
    steps:
    - name: Checkout artifactory repo
      uses: actions/checkout@v5

    - name: Clone private GitLab repository
      env:
        GITLAB_TOKEN: ${{ secrets.GITLAB_TOKEN }}
      run: |
        # Determine branch from webhook or manual input
        BRANCH="${{ github.event_name == 'repository_dispatch' && github.event.client_payload.ref || github.event.inputs.gitlab_branch || 'enterprise' }}"
        BRANCH=$(echo "$BRANCH" | sed 's|refs/heads/||')
        if [ "$BRANCH" = "null" ]; then
          BRANCH="enterprise"
        fi
        
        echo "Cloning branch: $BRANCH"
        git clone -b $BRANCH https://gitlab-ci-token:${GITLAB_TOKEN}@gitlab.com/chrislusf/seaweedfs.git seaweedfs-source
        
        cd seaweedfs-source
        echo "Cloned commit: $(git rev-parse HEAD)"
        echo "COMMIT_SHA=$(git rev-parse --short=8 HEAD)" >> $GITHUB_ENV

    - name: Set up Go 1.x
      uses: actions/setup-go@v6
      with:
        go-version-file: 'seaweedfs-source/go.mod'
      id: go

    - name: Get dependencies
      working-directory: seaweedfs-source/test/kafka
      run: |
        go mod download

    - name: Run Integration Tests
      working-directory: seaweedfs-source/test/kafka
      run: |
        # Higher limits for integration tests
        ulimit -n 1024 || echo "Warning: Could not set file descriptor limit"
        ulimit -u 200 || echo "Warning: Could not set process limit"
        go test -v -timeout 120s ./integration/... || echo "Integration tests completed with warnings"

  kafka-e2e-tests:
    name: Kafka End-to-End Tests
    runs-on: ubuntu-latest
    timeout-minutes: 25
    steps:
    - name: Checkout artifactory repo
      uses: actions/checkout@v5

    - name: Clone private GitLab repository
      env:
        GITLAB_TOKEN: ${{ secrets.GITLAB_TOKEN }}
      run: |
        # Determine branch from webhook or manual input
        BRANCH="${{ github.event_name == 'repository_dispatch' && github.event.client_payload.ref || github.event.inputs.gitlab_branch || 'enterprise' }}"
        BRANCH=$(echo "$BRANCH" | sed 's|refs/heads/||')
        if [ "$BRANCH" = "null" ]; then
          BRANCH="enterprise"
        fi
        
        echo "Cloning branch: $BRANCH"
        git clone -b $BRANCH https://gitlab-ci-token:${GITLAB_TOKEN}@gitlab.com/chrislusf/seaweedfs.git seaweedfs-source
        
        cd seaweedfs-source
        echo "Cloned commit: $(git rev-parse HEAD)"
        echo "COMMIT_SHA=$(git rev-parse --short=8 HEAD)" >> $GITHUB_ENV

    - name: Set up Go 1.x
      uses: actions/setup-go@v6
      with:
        go-version-file: 'seaweedfs-source/go.mod'
        cache: true
        cache-dependency-path: |
          **/go.sum
      id: go

    - name: Get dependencies
      working-directory: seaweedfs-source/test/kafka
      run: |
        # Use go mod download with timeout to prevent hanging
        timeout 90s go mod download || echo "Warning: Dependency download timed out, continuing with cached modules"

    - name: Build and start SeaweedFS MQ
      working-directory: seaweedfs-source
      run: |
        set -e
        # Build weed binary
        cd weed
        go build -o /usr/local/bin/weed .
        cd ..
        
        # Start SeaweedFS components with MQ brokers
        export WEED_DATA_DIR=/tmp/seaweedfs-e2e-$RANDOM
        mkdir -p "$WEED_DATA_DIR"
        
        # Start SeaweedFS server (master, volume, filer) with consistent IP advertising
        nohup weed -v 1 server \
          -ip="127.0.0.1" \
          -ip.bind="0.0.0.0" \
          -dir="$WEED_DATA_DIR" \
          -master.raftHashicorp \
          -master.port=9333 \
          -volume.port=8081 \
          -filer.port=8888 \
          -filer=true \
          -metricsPort=9325 \
          > /tmp/weed-server.log 2>&1 &
          
        # Wait for master to be ready
        for i in $(seq 1 30); do
          if curl -s http://127.0.0.1:9333/cluster/status >/dev/null; then
            echo "SeaweedFS master HTTP is up"; break
          fi
          echo "Waiting for SeaweedFS master HTTP... ($i/30)"; sleep 1
        done
        
        # Wait for master gRPC to be ready (this is what broker discovery uses)
        echo "Waiting for master gRPC port..."
        for i in $(seq 1 30); do
          if nc -z 127.0.0.1 19333 2>/dev/null; then
            echo "✓ SeaweedFS master gRPC is up (port 19333)"
            break
          fi
          echo "  Waiting for master gRPC... ($i/30)"; sleep 1
        done
        
        # Give server time to initialize all components including gRPC services
        echo "Waiting for SeaweedFS components to initialize..."
        sleep 15
        
        # Start MQ broker with maximum verbosity for debugging
        echo "Starting MQ broker..."
        nohup weed -v 3 mq.broker \
          -master="127.0.0.1:9333" \
          -ip="127.0.0.1" \
          -port=17777 \
          -logFlushInterval=0 \
          > /tmp/weed-mq-broker.log 2>&1 &
        
        # Wait for broker to be ready with better error reporting  
        sleep 15
        broker_ready=false
        for i in $(seq 1 20); do
          if nc -z 127.0.0.1 17777 2>/dev/null; then
            echo "SeaweedFS MQ broker is up"
            broker_ready=true
            break
          fi
          echo "Waiting for MQ broker... ($i/20)"; sleep 1
        done
        
        # Give broker additional time to register with master
        if [ "$broker_ready" = true ]; then
          echo "Allowing broker to register with master..."
          sleep 30
          
          # Check if broker is properly registered by querying cluster nodes
          echo "Cluster status after broker registration:"
          curl -s "http://127.0.0.1:9333/cluster/status" || echo "Could not check cluster status"
        fi
        
        # Check if broker failed to start and show logs
        if [ "$broker_ready" = false ]; then
          echo "ERROR: MQ broker failed to start. Broker logs:"
          cat /tmp/weed-mq-broker.log || echo "No broker logs found"
          echo "Server logs:"  
          tail -20 /tmp/weed-server.log || echo "No server logs found"
          exit 1
        fi

    - name: Run End-to-End Tests
      working-directory: seaweedfs-source/test/kafka
      run: |
        # Higher limits for E2E tests
        ulimit -n 1024 || echo "Warning: Could not set file descriptor limit"
        ulimit -u 200 || echo "Warning: Could not set process limit"
        
        # Allow additional time for all background processes to settle
        echo "Allowing additional settlement time for SeaweedFS ecosystem..."
        sleep 15
        
        # Run tests and capture result
        if ! go test -v -timeout 180s ./e2e/...; then
          echo "========================================="
          echo "Tests failed! Showing debug information:"
          echo "========================================="
          echo "Server logs (last 50 lines):"
          tail -50 /tmp/weed-server.log || echo "No server logs"
          echo "========================================="
          echo "Broker logs (last 50 lines):"
          tail -50 /tmp/weed-mq-broker.log || echo "No broker logs"
          echo "========================================="
          echo "Tests completed with failures"
        fi
      env:
        SEAWEEDFS_MASTERS: 127.0.0.1:9333

    - name: Upload test logs on failure
      if: failure()
      uses: actions/upload-artifact@v4
      with:
        name: kafka-e2e-logs-${{ env.COMMIT_SHA }}
        path: |
          /tmp/weed-server.log
          /tmp/weed-mq-broker.log
        retention-days: 3

  kafka-test-summary:
    name: Kafka Test Summary
    runs-on: ubuntu-latest
    needs: [kafka-unit-tests, kafka-integration-tests, kafka-e2e-tests]
    if: always()
    steps:
      - name: Generate Test Summary
        run: |
          echo "## 📨 Kafka Gateway Test Summary" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          echo "### GitLab Source" >> $GITHUB_STEP_SUMMARY
          echo "- **Repository**: https://gitlab.com/chrislusf/seaweedfs" >> $GITHUB_STEP_SUMMARY
          echo "- **Branch**: $(echo '${{ github.event.client_payload.ref || github.event.inputs.gitlab_branch || 'enterprise' }}' | sed 's|refs/heads/||')" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          echo "### Test Results" >> $GITHUB_STEP_SUMMARY
          echo "- **Unit Tests**: ${{ needs.kafka-unit-tests.result }}" >> $GITHUB_STEP_SUMMARY
          echo "- **Integration Tests**: ${{ needs.kafka-integration-tests.result }}" >> $GITHUB_STEP_SUMMARY
          echo "- **End-to-End Tests**: ${{ needs.kafka-e2e-tests.result }}" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          
          # Count successful tests
          SUCCESS_COUNT=0
          [ "${{ needs.kafka-unit-tests.result }}" = "success" ] && SUCCESS_COUNT=$((SUCCESS_COUNT + 1))
          [ "${{ needs.kafka-integration-tests.result }}" = "success" ] && SUCCESS_COUNT=$((SUCCESS_COUNT + 1))
          [ "${{ needs.kafka-e2e-tests.result }}" = "success" ] && SUCCESS_COUNT=$((SUCCESS_COUNT + 1))
          
          echo "### Summary" >> $GITHUB_STEP_SUMMARY
          echo "✅ **$SUCCESS_COUNT/3** test suites passed" >> $GITHUB_STEP_SUMMARY
          
          echo "" >> $GITHUB_STEP_SUMMARY
          echo "### Kafka Gateway Features" >> $GITHUB_STEP_SUMMARY
          echo "- 📨 **Kafka Protocol**: Compatible with Kafka clients" >> $GITHUB_STEP_SUMMARY
          echo "- 🔄 **Message Queuing**: Distributed message queue over SeaweedFS" >> $GITHUB_STEP_SUMMARY
          echo "- 👥 **Consumer Groups**: Multi-consumer message processing" >> $GITHUB_STEP_SUMMARY
          echo "- 🎯 **Topic Management**: Dynamic topic creation and management" >> $GITHUB_STEP_SUMMARY