test-hive.yml

name: Test Hive

on:
  push:
    branches:
      - main
      - 'release/**'
  pull_request:
    branches:
      - main
      - 'release/**'
      - performance
      - performance-stable
    types:
      - opened
      - reopened
      - synchronize
      - ready_for_review
  schedule:
    - cron: "0 05 * * *" # daily at 5 am UTC
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && format('pr-{0}', github.event.pull_request.number) || format('push-{0}-{1}', github.run_id, github.run_attempt) }}
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}

jobs:
  test-hive:
    if: ${{ github.event_name == 'push' || !github.event.pull_request.draft }}
    runs-on:
      group: hive
    strategy:
      fail-fast: false
      matrix:
        include:
          - sim: engine
            sim-limit: exchange-capabilities
            max-allowed-failures: 0
          - sim: engine
            sim-limit: withdrawals
            max-allowed-failures: 0
          - sim: engine
            sim-limit: cancun
            # 2 failures (not due to us, but due to Hive/Geth secondary client)
            # will remove once resolved by STEEL team (these 2 tests are failing on all clients)
            # see https://discord.com/channels/1359927674746835211/1410592782258540565/1462699469824065560
            # 3rd allowed failure: "Blob Transaction Ordering, Multiple Clients (Cancun)" is a
            # known flake under --sim.parallelism=8 due to timing/resource contention; passes
            # cleanly in isolation (--sim.parallelism=1)
            max-allowed-failures: 3
          - sim: engine
            sim-limit: api
            max-allowed-failures: 0
          - sim: engine
            sim-limit: auth
            max-allowed-failures: 0
          - sim: rpc
            sim-limit: compat
            max-allowed-failures: 23
    steps:
      - name: Checkout Erigon go.mod
        uses: actions/checkout@v6
        with:
          sparse-checkout: |
            go.mod
            .github/workflows/hive-versions.json
          sparse-checkout-cone-mode: false
          path: erigon-src

      - name: Read pinned versions
        id: hive-version
        run: |
          echo "ref=$(jq -r .hive_ref erigon-src/.github/workflows/hive-versions.json)" >> "$GITHUB_OUTPUT"
          echo "execution_apis_ref=$(jq -r '.execution_apis_ref // empty' erigon-src/.github/workflows/hive-versions.json)" >> "$GITHUB_OUTPUT"

      - name: Checkout Hive
        uses: actions/checkout@v6
        with:
          repository: ethereum/hive
          # version hive and update periodically/on-demand to prevent upstream changes in Hive affecting us with red CI
          ref: ${{ steps.hive-version.outputs.ref }}
          path: hive

      - name: Setup go env and cache
        uses: actions/setup-go@v6
        with:
          go-version: '>=1.25'
          go-version-file: 'hive/go.mod'

      - name: Conditional Docker Login
        # Only login if we can. Workflow works without it but we want to avoid
        # rate limiting by Docker Hub when possible. External repos don't
        # have access to our Docker secrets.
        # continue-on-error: transient Docker Hub network timeouts should not
        # abort the entire workflow — the run proceeds without login (unlogged pull).
        if: |
          github.repository == 'erigontech/erigon' && 
          github.actor != 'dependabot[bot]' &&
          (github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork)
        continue-on-error: true
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.ORG_DOCKERHUB_ERIGONTECH_USERNAME }}
          password: ${{ secrets.ORG_DOCKERHUB_ERIGONTECH_TOKEN }}

      # Targeting the clients/erigon/Dockerfile.git in the Hive directory -
      # this builds the container from github and uses it for tests
      - name: Get dependencies and build hive
        env:
          SOURCE_REPO: ${{ github.event.pull_request.head.repo.full_name || github.repository }}
        run: |
          cd hive
          go get . >> buildlogs.log
          rm clients/erigon/Dockerfile
          mv clients/erigon/Dockerfile.git clients/erigon/Dockerfile
          branch_name=$(echo ${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}} | sed 's/[&/\]/\\&/g')
          echo Building Hive with Erigon repo - ${SOURCE_REPO}, branch - $branch_name
          sed -i "s|^ARG github=erigontech/erigon$|ARG github=${SOURCE_REPO}|" clients/erigon/Dockerfile
          sed -i "s/^ARG tag=main$/ARG tag=${branch_name}/" clients/erigon/Dockerfile
          go_version=$(go mod edit -json ../erigon-src/go.mod | jq -r .Go)
          echo "Patching builder Go version to ${go_version}"
          sed -i "s|golang:[0-9.]*-alpine|golang:${go_version}-alpine|" clients/erigon/Dockerfile
          # Pin the execution-apis ref used by the rpc-compat simulator so that
          # upstream test additions don't break CI unexpectedly.
          execution_apis_ref="${{ steps.hive-version.outputs.execution_apis_ref }}"
          if [ -n "$execution_apis_ref" ]; then
            echo "Pinning rpc-compat execution-apis ref to ${execution_apis_ref}"
            sed -i "s/^ARG branch=main$/ARG branch=${execution_apis_ref}/" simulators/ethereum/rpc-compat/Dockerfile
          fi
          go build . >> buildlogs.log
      # Depends on the last line of hive output that prints the number of suites, tests and failed
      # Currently, we fail even if suites and tests are too few, indicating the tests did not run
      # We also fail if more than half the tests fail
      - name: Run hive tests and parse output
        run: |
          cd hive
          run_suite() {
            if [ $# -ne 3 ]; then
              echo "Error: run_suite requires exactly 3 parameters"
              echo "Usage: run_suite <sim> <sim.limit> <max_allowed_failures>"
              echo "Provided: $# parameters"
              exit 1
            fi
            echo -e "\n\n============================================================"
            echo "Running test: ${1}-${2}"
            echo -e "\n"
            ./hive -docker.auth --sim ethereum/"${1}" --sim.limit="${2}" --sim.parallelism=8 --client erigon 2>&1 | tee output.log || {
              if [ $? -gt 0 ]; then
                echo "Exitcode gt 0"
              fi
            }
            status_line=$(tail -2 output.log | head -1 | sed -r "s/\x1B\[[0-9;]*[a-zA-Z]//g")
            suites=$(echo "$status_line" | sed -n 's/.*suites=\([0-9]*\).*/\1/p')
            if [ -z "$suites" ]; then
              status_line=$(tail -1 output.log | sed -r "s/\x1B\[[0-9;]*[a-zA-Z]//g")
              suites=$(echo "$status_line" | sed -n 's/.*suites=\([0-9]*\).*/\1/p')
            fi 
            tests=$(echo "$status_line" | sed -n 's/.*tests=\([0-9]*\).*/\1/p')
            failed=$(echo "$status_line" | sed -n 's/.*failed=\([0-9]*\).*/\1/p')

            echo -e "\n"
            echo "-----------   Results for ${1}-${2}    -----------" 
            echo "Tests: $tests, Failed: $failed"
            echo -e "\n\n============================================================"

            if (( tests < 4 )); then
              echo "Too few tests run for suite ${1}-${2} - ${tests} tests"
              echo "failed" > failed.log
              exit 1
            fi
            max_allowed_failures="${3}"
            if (( failed > max_allowed_failures )); then
              echo "Too many failures for suite ${1}-${2} - ${failed} failed out of ${tests}"
              echo "failed" > failed.log
              exit 1
            fi
          }
          run_suite ${{ matrix.sim }} ${{ matrix.sim-limit }} ${{ matrix.max-allowed-failures }}
        continue-on-error: true

      - name: Upload output log
        uses: actions/upload-artifact@v6
        with:
          name: hive-workspace-log-${{ matrix.sim }}-${{ matrix.sim-limit }}
          path: hive/workspace/logs
        continue-on-error: true

      - name: Check for failures
        run: |
          if grep -q "failed" hive/failed.log; then
            echo "One or more tests failed."
            exit 1
          fi
          echo "All tests passed successfully."

      # This step is not required UNTIL the github-managed runners are dismissed in favor of self-hosted ones (which is planned)
      # So it is good to PROACTIVELY run it (it should not cause any issues within github-managed runners either)
      - name: Remove Hive directory
        run: |
          echo "Removing the Hive directory..."
          rm -rf hive
        if: always()

      # This step is not required UNTIL the github-managed runners are dismissed in favor of self-hosted ones (which is planned)
      # So it is good to PROACTIVELY run it (it should not cause any issues within github-managed runners either)  
      - name: Prune docker
        run: |
          echo "Pruning docker..."
          docker system prune -af --volumes
        if: always()