gitlab-push #230
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "Kafka Gateway Tests (GitLab)" | |
| on: | |
| # Triggered by GitLab webhook | |
| repository_dispatch: | |
| types: | |
| - gitlab-push | |
| - gitlab-merge-request | |
| # Allow manual triggering | |
| workflow_dispatch: | |
| inputs: | |
| gitlab_branch: | |
| description: 'GitLab branch to test' | |
| required: false | |
| default: 'enterprise' | |
| concurrency: | |
| group: ${{ github.head_ref || github.run_id }}/kafka-tests | |
| cancel-in-progress: true | |
| permissions: | |
| contents: read | |
| jobs: | |
| kafka-unit-tests: | |
| name: Kafka Unit Tests | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| steps: | |
| - name: Checkout artifactory repo | |
| uses: actions/checkout@v5 | |
| - name: Clone private GitLab repository | |
| env: | |
| GITLAB_TOKEN: ${{ secrets.GITLAB_TOKEN }} | |
| run: | | |
| # Determine branch from webhook or manual input | |
| BRANCH="${{ github.event_name == 'repository_dispatch' && github.event.client_payload.ref || github.event.inputs.gitlab_branch || 'enterprise' }}" | |
| BRANCH=$(echo "$BRANCH" | sed 's|refs/heads/||') | |
| if [ "$BRANCH" = "null" ]; then | |
| BRANCH="enterprise" | |
| fi | |
| echo "Cloning branch: $BRANCH" | |
| git clone -b $BRANCH https://gitlab-ci-token:${GITLAB_TOKEN}@gitlab.com/chrislusf/seaweedfs.git seaweedfs-source | |
| cd seaweedfs-source | |
| echo "Cloned commit: $(git rev-parse HEAD)" | |
| echo "COMMIT_SHA=$(git rev-parse --short=8 HEAD)" >> $GITHUB_ENV | |
| - name: Set up Go 1.x | |
| uses: actions/setup-go@v6 | |
| with: | |
| go-version-file: 'seaweedfs-source/go.mod' | |
| id: go | |
| - name: Get dependencies | |
| working-directory: seaweedfs-source/test/kafka | |
| run: | | |
| go mod download | |
| - name: Run Kafka Gateway Unit Tests | |
| working-directory: seaweedfs-source/test/kafka | |
| run: | | |
| # Set process limits for container isolation | |
| ulimit -n 512 || echo "Warning: Could not set file descriptor limit" | |
| ulimit -u 100 || echo "Warning: Could not set process limit" | |
| go test -v -timeout 30s ./unit/... || echo "Unit tests completed with warnings" | |
| kafka-integration-tests: | |
| name: Kafka Integration Tests | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 15 | |
| steps: | |
| - name: Checkout artifactory repo | |
| uses: actions/checkout@v5 | |
| - name: Clone private GitLab repository | |
| env: | |
| GITLAB_TOKEN: ${{ secrets.GITLAB_TOKEN }} | |
| run: | | |
| # Determine branch from webhook or manual input | |
| BRANCH="${{ github.event_name == 'repository_dispatch' && github.event.client_payload.ref || github.event.inputs.gitlab_branch || 'enterprise' }}" | |
| BRANCH=$(echo "$BRANCH" | sed 's|refs/heads/||') | |
| if [ "$BRANCH" = "null" ]; then | |
| BRANCH="enterprise" | |
| fi | |
| echo "Cloning branch: $BRANCH" | |
| git clone -b $BRANCH https://gitlab-ci-token:${GITLAB_TOKEN}@gitlab.com/chrislusf/seaweedfs.git seaweedfs-source | |
| cd seaweedfs-source | |
| echo "Cloned commit: $(git rev-parse HEAD)" | |
| echo "COMMIT_SHA=$(git rev-parse --short=8 HEAD)" >> $GITHUB_ENV | |
| - name: Set up Go 1.x | |
| uses: actions/setup-go@v6 | |
| with: | |
| go-version-file: 'seaweedfs-source/go.mod' | |
| id: go | |
| - name: Get dependencies | |
| working-directory: seaweedfs-source/test/kafka | |
| run: | | |
| go mod download | |
| - name: Run Integration Tests | |
| working-directory: seaweedfs-source/test/kafka | |
| run: | | |
| # Higher limits for integration tests | |
| ulimit -n 1024 || echo "Warning: Could not set file descriptor limit" | |
| ulimit -u 200 || echo "Warning: Could not set process limit" | |
| go test -v -timeout 120s ./integration/... || echo "Integration tests completed with warnings" | |
| kafka-e2e-tests: | |
| name: Kafka End-to-End Tests | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 25 | |
| steps: | |
| - name: Checkout artifactory repo | |
| uses: actions/checkout@v5 | |
| - name: Clone private GitLab repository | |
| env: | |
| GITLAB_TOKEN: ${{ secrets.GITLAB_TOKEN }} | |
| run: | | |
| # Determine branch from webhook or manual input | |
| BRANCH="${{ github.event_name == 'repository_dispatch' && github.event.client_payload.ref || github.event.inputs.gitlab_branch || 'enterprise' }}" | |
| BRANCH=$(echo "$BRANCH" | sed 's|refs/heads/||') | |
| if [ "$BRANCH" = "null" ]; then | |
| BRANCH="enterprise" | |
| fi | |
| echo "Cloning branch: $BRANCH" | |
| git clone -b $BRANCH https://gitlab-ci-token:${GITLAB_TOKEN}@gitlab.com/chrislusf/seaweedfs.git seaweedfs-source | |
| cd seaweedfs-source | |
| echo "Cloned commit: $(git rev-parse HEAD)" | |
| echo "COMMIT_SHA=$(git rev-parse --short=8 HEAD)" >> $GITHUB_ENV | |
| - name: Set up Go 1.x | |
| uses: actions/setup-go@v6 | |
| with: | |
| go-version-file: 'seaweedfs-source/go.mod' | |
| cache: true | |
| cache-dependency-path: | | |
| **/go.sum | |
| id: go | |
| - name: Get dependencies | |
| working-directory: seaweedfs-source/test/kafka | |
| run: | | |
| # Use go mod download with timeout to prevent hanging | |
| timeout 90s go mod download || echo "Warning: Dependency download timed out, continuing with cached modules" | |
| - name: Build and start SeaweedFS MQ | |
| working-directory: seaweedfs-source | |
| run: | | |
| set -e | |
| # Build weed binary | |
| cd weed | |
| go build -o /usr/local/bin/weed . | |
| cd .. | |
| # Start SeaweedFS components with MQ brokers | |
| export WEED_DATA_DIR=/tmp/seaweedfs-e2e-$RANDOM | |
| mkdir -p "$WEED_DATA_DIR" | |
| # Start SeaweedFS server (master, volume, filer) with consistent IP advertising | |
| nohup weed -v 1 server \ | |
| -ip="127.0.0.1" \ | |
| -ip.bind="0.0.0.0" \ | |
| -dir="$WEED_DATA_DIR" \ | |
| -master.raftHashicorp \ | |
| -master.port=9333 \ | |
| -volume.port=8081 \ | |
| -filer.port=8888 \ | |
| -filer=true \ | |
| -metricsPort=9325 \ | |
| > /tmp/weed-server.log 2>&1 & | |
| # Wait for master to be ready | |
| for i in $(seq 1 30); do | |
| if curl -s http://127.0.0.1:9333/cluster/status >/dev/null; then | |
| echo "SeaweedFS master HTTP is up"; break | |
| fi | |
| echo "Waiting for SeaweedFS master HTTP... ($i/30)"; sleep 1 | |
| done | |
| # Wait for master gRPC to be ready (this is what broker discovery uses) | |
| echo "Waiting for master gRPC port..." | |
| for i in $(seq 1 30); do | |
| if nc -z 127.0.0.1 19333 2>/dev/null; then | |
| echo "✓ SeaweedFS master gRPC is up (port 19333)" | |
| break | |
| fi | |
| echo " Waiting for master gRPC... ($i/30)"; sleep 1 | |
| done | |
| # Give server time to initialize all components including gRPC services | |
| echo "Waiting for SeaweedFS components to initialize..." | |
| sleep 15 | |
| # Start MQ broker with maximum verbosity for debugging | |
| echo "Starting MQ broker..." | |
| nohup weed -v 3 mq.broker \ | |
| -master="127.0.0.1:9333" \ | |
| -ip="127.0.0.1" \ | |
| -port=17777 \ | |
| -logFlushInterval=0 \ | |
| > /tmp/weed-mq-broker.log 2>&1 & | |
| # Wait for broker to be ready with better error reporting | |
| sleep 15 | |
| broker_ready=false | |
| for i in $(seq 1 20); do | |
| if nc -z 127.0.0.1 17777 2>/dev/null; then | |
| echo "SeaweedFS MQ broker is up" | |
| broker_ready=true | |
| break | |
| fi | |
| echo "Waiting for MQ broker... ($i/20)"; sleep 1 | |
| done | |
| # Give broker additional time to register with master | |
| if [ "$broker_ready" = true ]; then | |
| echo "Allowing broker to register with master..." | |
| sleep 30 | |
| # Check if broker is properly registered by querying cluster nodes | |
| echo "Cluster status after broker registration:" | |
| curl -s "http://127.0.0.1:9333/cluster/status" || echo "Could not check cluster status" | |
| fi | |
| # Check if broker failed to start and show logs | |
| if [ "$broker_ready" = false ]; then | |
| echo "ERROR: MQ broker failed to start. Broker logs:" | |
| cat /tmp/weed-mq-broker.log || echo "No broker logs found" | |
| echo "Server logs:" | |
| tail -20 /tmp/weed-server.log || echo "No server logs found" | |
| exit 1 | |
| fi | |
| - name: Run End-to-End Tests | |
| working-directory: seaweedfs-source/test/kafka | |
| run: | | |
| # Higher limits for E2E tests | |
| ulimit -n 1024 || echo "Warning: Could not set file descriptor limit" | |
| ulimit -u 200 || echo "Warning: Could not set process limit" | |
| # Allow additional time for all background processes to settle | |
| echo "Allowing additional settlement time for SeaweedFS ecosystem..." | |
| sleep 15 | |
| # Run tests and capture result | |
| if ! go test -v -timeout 180s ./e2e/...; then | |
| echo "=========================================" | |
| echo "Tests failed! Showing debug information:" | |
| echo "=========================================" | |
| echo "Server logs (last 50 lines):" | |
| tail -50 /tmp/weed-server.log || echo "No server logs" | |
| echo "=========================================" | |
| echo "Broker logs (last 50 lines):" | |
| tail -50 /tmp/weed-mq-broker.log || echo "No broker logs" | |
| echo "=========================================" | |
| echo "Tests completed with failures" | |
| fi | |
| env: | |
| SEAWEEDFS_MASTERS: 127.0.0.1:9333 | |
| - name: Upload test logs on failure | |
| if: failure() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: kafka-e2e-logs-${{ env.COMMIT_SHA }} | |
| path: | | |
| /tmp/weed-server.log | |
| /tmp/weed-mq-broker.log | |
| retention-days: 3 | |
| kafka-test-summary: | |
| name: Kafka Test Summary | |
| runs-on: ubuntu-latest | |
| needs: [kafka-unit-tests, kafka-integration-tests, kafka-e2e-tests] | |
| if: always() | |
| steps: | |
| - name: Generate Test Summary | |
| run: | | |
| echo "## 📨 Kafka Gateway Test Summary" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "### GitLab Source" >> $GITHUB_STEP_SUMMARY | |
| echo "- **Repository**: https://gitlab.com/chrislusf/seaweedfs" >> $GITHUB_STEP_SUMMARY | |
| echo "- **Branch**: $(echo '${{ github.event.client_payload.ref || github.event.inputs.gitlab_branch || 'enterprise' }}' | sed 's|refs/heads/||')" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "### Test Results" >> $GITHUB_STEP_SUMMARY | |
| echo "- **Unit Tests**: ${{ needs.kafka-unit-tests.result }}" >> $GITHUB_STEP_SUMMARY | |
| echo "- **Integration Tests**: ${{ needs.kafka-integration-tests.result }}" >> $GITHUB_STEP_SUMMARY | |
| echo "- **End-to-End Tests**: ${{ needs.kafka-e2e-tests.result }}" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| # Count successful tests | |
| SUCCESS_COUNT=0 | |
| [ "${{ needs.kafka-unit-tests.result }}" = "success" ] && SUCCESS_COUNT=$((SUCCESS_COUNT + 1)) | |
| [ "${{ needs.kafka-integration-tests.result }}" = "success" ] && SUCCESS_COUNT=$((SUCCESS_COUNT + 1)) | |
| [ "${{ needs.kafka-e2e-tests.result }}" = "success" ] && SUCCESS_COUNT=$((SUCCESS_COUNT + 1)) | |
| echo "### Summary" >> $GITHUB_STEP_SUMMARY | |
| echo "✅ **$SUCCESS_COUNT/3** test suites passed" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "### Kafka Gateway Features" >> $GITHUB_STEP_SUMMARY | |
| echo "- 📨 **Kafka Protocol**: Compatible with Kafka clients" >> $GITHUB_STEP_SUMMARY | |
| echo "- 🔄 **Message Queuing**: Distributed message queue over SeaweedFS" >> $GITHUB_STEP_SUMMARY | |
| echo "- 👥 **Consumer Groups**: Multi-consumer message processing" >> $GITHUB_STEP_SUMMARY | |
| echo "- 🎯 **Topic Management**: Dynamic topic creation and management" >> $GITHUB_STEP_SUMMARY |