test-kurtosis-gloas.yml

name: Kurtosis GLOAS Tests

env:
  DOCKERHUB_REPOSITORY: "erigontech/erigon"
  APP_REPO: "erigontech/erigon"
  # ASSERTOOR_IMAGE and GENESIS_GENERATOR_IMAGE are loaded at job start from the
  # gloas-*.io files (single source of truth) by the "Load image versions" step, so
  # the cache can't drift from what kurtosis runs. Mutable lighthouse/prysm devnet
  # client images are pulled live (not cached) to avoid serving a stale client.
  # Kurtosis CLI pinned so its infra images (engine/core/files-artifacts-expander
  # are tagged with the CLI version) can be cached too. The vector and fluent-bit
  # tags are dictated by the Kurtosis release — sync them when bumping (see
  # logs_aggregator_functions and logs_collector_functions consts in
  # kurtosis-tech/kurtosis).
  KURTOSIS_VERSION: "1.20.0"
  KURTOSIS_VECTOR_IMAGE: "timberio/vector:0.45.0-debian"
  KURTOSIS_FLUENTBIT_IMAGE: "fluent/fluent-bit:4.0.0"
  # Engine-bootstrap helpers (logs-aggregator healthcheck, reverse proxy,
  # volume init) — also dictated by the Kurtosis release; sync when bumping.
  # curl-jq is untagged upstream, so the cache intentionally serves the
  # `latest` digest from the last warm rather than tracking live `latest`.
  # Without these cached, `kurtosis engine start` still needs Docker Hub and
  # dies on registry outages despite all the caching above.
  KURTOSIS_CURL_JQ_IMAGE: "badouralix/curl-jq:latest"
  KURTOSIS_TRAEFIK_IMAGE: "traefik:2.10.6"
  KURTOSIS_ALPINE_IMAGE: "alpine:3.17"
  # BuildKit image booted by docker/setup-buildx-action (pinned via driver-opts).
  # Cached and pre-loaded so buildx can fall back to the local copy when its
  # Docker Hub pull fails.
  BUILDKIT_IMAGE: "moby/buildkit:v0.30.0"
  # eth2-val-tools is the validator-keystore generator pulled during `kurtosis run`;
  # it publishes only :latest, so the cache serves the `latest` digest from the last
  # warm rather than tracking live `latest`.
  ETH2_VAL_TOOLS_IMAGE: "protolambda/eth2-val-tools:latest"

on:
  pull_request:
    branches:
      - main
      - 'release/**'
    types:
      - opened
      - reopened
      - synchronize
      - ready_for_review
  workflow_dispatch:
  workflow_call:
    inputs:
      cl-images-only:
        description: "Only warm the third-party CL/kurtosis image cache; skip the tests"
        type: boolean
        default: false

concurrency:
  group: >-
    ${{
      github.event_name == 'pull_request' &&
      format('{0}-{1}', github.workflow, github.head_ref) ||
      format('{0}-{1}', github.workflow, github.run_id)
    }}
  cancel-in-progress: true

permissions:
  contents: read

jobs:
  # Warms the docker-cl-* and docker-buildkit-* caches on the default branch
  # (where this workflow's PR runs restore them) via a dedicated workflow on a
  # paths filter + daily schedule — the caches only change when the pinned
  # image versions in this file change. lookup-only makes it a no-op when the
  # cache already exists.
  warm-third-party-images:
    if: ${{ inputs.cl-images-only }}
    runs-on: ubuntu-latest
    steps:
      - name: Conditional Docker Login
        if: |
          github.repository == 'erigontech/erigon' &&
          github.actor != 'dependabot[bot]' &&
          !github.event.pull_request.head.repo.fork
        continue-on-error: true
        uses: docker/login-action@v4
        with:
          username: ${{ secrets.DOCKERHUB_PULL_USERNAME }}
          password: ${{ secrets.DOCKERHUB_PULL_TOKEN }}

      - name: Checkout kurtosis .io files
        uses: actions/checkout@v7
        with:
          persist-credentials: false
          sparse-checkout: .github/workflows/kurtosis

      - name: Load image versions from the .io files (single source of truth)
        run: |
          io=.github/workflows/kurtosis/gloas-caplin-mixed.io
          assertoor=$(yq -e '.assertoor_params.image' "$io")
          genesis=$(yq -e '.ethereum_genesis_generator_params.image' "$io")
          # One cache (keyed off $io) is shared by every gloas suite, so all
          # gloas-*.io must pin the same assertoor + genesis — fail if any diverges.
          for f in .github/workflows/kurtosis/gloas-*.io; do
            [ "$(yq -e '.assertoor_params.image' "$f")" = "$assertoor" ] \
              && [ "$(yq -e '.ethereum_genesis_generator_params.image' "$f")" = "$genesis" ] \
              || { echo "::error::$f disagrees with $io on the assertoor/genesis image; reconcile them or extend the gloas cache key"; exit 1; }
          done
          {
            echo "ASSERTOOR_IMAGE=$assertoor"
            echo "GENESIS_GENERATOR_IMAGE=$genesis"
          } >> "$GITHUB_ENV"

      - name: Check whether third-party containers are already cached
        id: cache-cl-images
        uses: actions/cache/restore@v6
        with:
          path: /tmp/docker-cache
          key: docker-cl-${{ env.ASSERTOOR_IMAGE }}-kurtosis-${{ env.KURTOSIS_VERSION }}-${{ env.KURTOSIS_VECTOR_IMAGE }}-${{ env.KURTOSIS_FLUENTBIT_IMAGE }}-${{ env.KURTOSIS_CURL_JQ_IMAGE }}-${{ env.KURTOSIS_TRAEFIK_IMAGE }}-${{ env.KURTOSIS_ALPINE_IMAGE }}-${{ env.GENESIS_GENERATOR_IMAGE }}-${{ env.ETH2_VAL_TOOLS_IMAGE }}
          lookup-only: true

      - name: Pull third-party containers
        if: steps.cache-cl-images.outputs.cache-hit != 'true'
        run: |
          mkdir -p /tmp/docker-cache
          pull() {
            for n in 1 2 3; do
              if docker pull "$1"; then return 0; fi
              echo "docker pull $1 failed (attempt $n of 3)"
              sleep $((10 * n))
            done
            return 1
          }
          pull "${ASSERTOOR_IMAGE}"
          pull "kurtosistech/engine:${KURTOSIS_VERSION}"
          pull "kurtosistech/core:${KURTOSIS_VERSION}"
          pull "kurtosistech/files-artifacts-expander:${KURTOSIS_VERSION}"
          pull "${KURTOSIS_VECTOR_IMAGE}"
          pull "${KURTOSIS_FLUENTBIT_IMAGE}"
          pull "${KURTOSIS_CURL_JQ_IMAGE}"
          pull "${KURTOSIS_TRAEFIK_IMAGE}"
          pull "${KURTOSIS_ALPINE_IMAGE}"
          pull "${GENESIS_GENERATOR_IMAGE}"
          pull "${ETH2_VAL_TOOLS_IMAGE}"
          docker save "${ASSERTOOR_IMAGE}" -o /tmp/docker-cache/assertoor.tar
          docker save "kurtosistech/engine:${KURTOSIS_VERSION}" -o /tmp/docker-cache/kurtosis-engine.tar
          docker save "kurtosistech/core:${KURTOSIS_VERSION}" -o /tmp/docker-cache/kurtosis-core.tar
          docker save "kurtosistech/files-artifacts-expander:${KURTOSIS_VERSION}" -o /tmp/docker-cache/kurtosis-expander.tar
          docker save "${KURTOSIS_VECTOR_IMAGE}" -o /tmp/docker-cache/vector.tar
          docker save "${KURTOSIS_FLUENTBIT_IMAGE}" -o /tmp/docker-cache/fluentbit.tar
          docker save "${KURTOSIS_CURL_JQ_IMAGE}" -o /tmp/docker-cache/curl-jq.tar
          docker save "${KURTOSIS_TRAEFIK_IMAGE}" -o /tmp/docker-cache/traefik.tar
          docker save "${KURTOSIS_ALPINE_IMAGE}" -o /tmp/docker-cache/alpine.tar
          docker save "${GENESIS_GENERATOR_IMAGE}" -o /tmp/docker-cache/genesis-generator.tar
          docker save "${ETH2_VAL_TOOLS_IMAGE}" -o /tmp/docker-cache/eth2-val-tools.tar

      - name: Save third-party containers to cache
        if: steps.cache-cl-images.outputs.cache-hit != 'true'
        uses: actions/cache/save@v6
        with:
          path: /tmp/docker-cache
          key: docker-cl-${{ env.ASSERTOOR_IMAGE }}-kurtosis-${{ env.KURTOSIS_VERSION }}-${{ env.KURTOSIS_VECTOR_IMAGE }}-${{ env.KURTOSIS_FLUENTBIT_IMAGE }}-${{ env.KURTOSIS_CURL_JQ_IMAGE }}-${{ env.KURTOSIS_TRAEFIK_IMAGE }}-${{ env.KURTOSIS_ALPINE_IMAGE }}-${{ env.GENESIS_GENERATOR_IMAGE }}-${{ env.ETH2_VAL_TOOLS_IMAGE }}
      - name: Check whether buildkit image is already cached
        id: cache-buildkit-image
        uses: actions/cache/restore@v6
        with:
          path: /tmp/docker-cache-buildkit
          key: docker-buildkit-${{ env.BUILDKIT_IMAGE }}
          lookup-only: true

      - name: Pull buildkit image
        if: steps.cache-buildkit-image.outputs.cache-hit != 'true'
        run: |
          mkdir -p /tmp/docker-cache-buildkit
          pull() {
            for n in 1 2 3; do
              if docker pull "$1"; then return 0; fi
              echo "docker pull $1 failed (attempt $n of 3)"
              sleep $((10 * n))
            done
            return 1
          }
          pull "${BUILDKIT_IMAGE}"
          docker save "${BUILDKIT_IMAGE}" -o /tmp/docker-cache-buildkit/buildkit.tar

      - name: Save buildkit image to cache
        if: steps.cache-buildkit-image.outputs.cache-hit != 'true'
        uses: actions/cache/save@v6
        with:
          path: /tmp/docker-cache-buildkit
          key: docker-buildkit-${{ env.BUILDKIT_IMAGE }}

  gloas_test:
    name: gloas_${{ matrix.suite }}_test
    if: ${{ !inputs.cl-images-only && !github.event.pull_request.draft }}
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        include:
          - suite: gloas-caplin-mixed
            package_args: .github/workflows/kurtosis/gloas-caplin-mixed.io
            ethereum_package_branch: "6.1.0"
            test_timeout_minutes: 20
          - suite: gloas-three-cl-mixed
            package_args: .github/workflows/kurtosis/gloas-three-cl-mixed.io
            ethereum_package_branch: "6.1.0"
            test_timeout_minutes: 20

    steps:
      - name: Fast checkout git repository
        uses: actions/checkout@v7
        with:
          persist-credentials: false

      - name: Load image versions from the .io files (single source of truth)
        run: |
          io=.github/workflows/kurtosis/gloas-caplin-mixed.io
          assertoor=$(yq -e '.assertoor_params.image' "$io")
          genesis=$(yq -e '.ethereum_genesis_generator_params.image' "$io")
          # One cache (keyed off $io) is shared by every gloas suite, so all
          # gloas-*.io must pin the same assertoor + genesis — fail if any diverges.
          for f in .github/workflows/kurtosis/gloas-*.io; do
            [ "$(yq -e '.assertoor_params.image' "$f")" = "$assertoor" ] \
              && [ "$(yq -e '.ethereum_genesis_generator_params.image' "$f")" = "$genesis" ] \
              || { echo "::error::$f disagrees with $io on the assertoor/genesis image; reconcile them or extend the gloas cache key"; exit 1; }
          done
          {
            echo "ASSERTOOR_IMAGE=$assertoor"
            echo "GENESIS_GENERATOR_IMAGE=$genesis"
          } >> "$GITHUB_ENV"

      - name: Conditional Docker Login
        if: |
          github.repository == 'erigontech/erigon' &&
          github.actor != 'dependabot[bot]' &&
          !github.event.pull_request.head.repo.fork
        continue-on-error: true
        uses: docker/login-action@v4
        with:
          username: ${{ secrets.DOCKERHUB_PULL_USERNAME }}
          password: ${{ secrets.DOCKERHUB_PULL_TOKEN }}

      - name: Restore cached buildkit image
        id: cache-buildkit-image
        uses: actions/cache/restore@v6
        with:
          path: /tmp/docker-cache-buildkit
          key: docker-buildkit-${{ env.BUILDKIT_IMAGE }}

      - name: Load cached buildkit image into daemon
        if: steps.cache-buildkit-image.outputs.cache-hit == 'true'
        run: docker load -i /tmp/docker-cache-buildkit/buildkit.tar

      # Best-effort: buildx pulls the image itself during bootstrap; this step
      # only seeds the cache that future runs fall back on.
      - name: Pull buildkit image for cache
        id: pull-buildkit-image
        if: steps.cache-buildkit-image.outputs.cache-hit != 'true'
        continue-on-error: true
        run: |
          pull() {
            for n in 1 2 3; do
              if docker pull "$1"; then return 0; fi
              echo "docker pull $1 failed (attempt $n of 3)"
              sleep $((10 * n))
            done
            return 1
          }
          pull "${BUILDKIT_IMAGE}"
          mkdir -p /tmp/docker-cache-buildkit
          docker save "${BUILDKIT_IMAGE}" -o /tmp/docker-cache-buildkit/buildkit.tar

      - name: Save buildkit image to cache
        if: steps.cache-buildkit-image.outputs.cache-hit != 'true' && steps.pull-buildkit-image.outcome == 'success' && github.event_name != 'pull_request'
        uses: actions/cache/save@v6
        with:
          path: /tmp/docker-cache-buildkit
          key: docker-buildkit-${{ env.BUILDKIT_IMAGE }}

      # setup-buildx boots BuildKit by pulling the pinned image from Docker Hub,
      # falling back to the pre-loaded local copy when the registry is down.
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v4
        with:
          driver-opts: image=${{ env.BUILDKIT_IMAGE }}

      - name: Restore cached third-party containers
        id: cache-cl-images
        uses: actions/cache/restore@v6
        with:
          path: /tmp/docker-cache
          key: docker-cl-${{ env.ASSERTOOR_IMAGE }}-kurtosis-${{ env.KURTOSIS_VERSION }}-${{ env.KURTOSIS_VECTOR_IMAGE }}-${{ env.KURTOSIS_FLUENTBIT_IMAGE }}-${{ env.KURTOSIS_CURL_JQ_IMAGE }}-${{ env.KURTOSIS_TRAEFIK_IMAGE }}-${{ env.KURTOSIS_ALPINE_IMAGE }}-${{ env.GENESIS_GENERATOR_IMAGE }}-${{ env.ETH2_VAL_TOOLS_IMAGE }}
      - name: Load cached containers into daemon
        if: steps.cache-cl-images.outputs.cache-hit == 'true'
        run: |
          docker load -i /tmp/docker-cache/assertoor.tar
          docker load -i /tmp/docker-cache/kurtosis-engine.tar
          docker load -i /tmp/docker-cache/kurtosis-core.tar
          docker load -i /tmp/docker-cache/kurtosis-expander.tar
          docker load -i /tmp/docker-cache/vector.tar
          docker load -i /tmp/docker-cache/fluentbit.tar
          docker load -i /tmp/docker-cache/curl-jq.tar
          docker load -i /tmp/docker-cache/traefik.tar
          docker load -i /tmp/docker-cache/alpine.tar
          docker load -i /tmp/docker-cache/genesis-generator.tar
          docker load -i /tmp/docker-cache/eth2-val-tools.tar

      - name: Pull third-party containers and save to cache
        if: steps.cache-cl-images.outputs.cache-hit != 'true'
        run: |
          mkdir -p /tmp/docker-cache
          pull() {
            for n in 1 2 3; do
              if docker pull "$1"; then return 0; fi
              echo "docker pull $1 failed (attempt $n of 3)"
              sleep $((10 * n))
            done
            return 1
          }
          pull "${ASSERTOOR_IMAGE}"
          pull "kurtosistech/engine:${KURTOSIS_VERSION}"
          pull "kurtosistech/core:${KURTOSIS_VERSION}"
          pull "kurtosistech/files-artifacts-expander:${KURTOSIS_VERSION}"
          pull "${KURTOSIS_VECTOR_IMAGE}"
          pull "${KURTOSIS_FLUENTBIT_IMAGE}"
          pull "${KURTOSIS_CURL_JQ_IMAGE}"
          pull "${KURTOSIS_TRAEFIK_IMAGE}"
          pull "${KURTOSIS_ALPINE_IMAGE}"
          pull "${GENESIS_GENERATOR_IMAGE}"
          pull "${ETH2_VAL_TOOLS_IMAGE}"
          docker save "${ASSERTOOR_IMAGE}" -o /tmp/docker-cache/assertoor.tar
          docker save "kurtosistech/engine:${KURTOSIS_VERSION}" -o /tmp/docker-cache/kurtosis-engine.tar
          docker save "kurtosistech/core:${KURTOSIS_VERSION}" -o /tmp/docker-cache/kurtosis-core.tar
          docker save "kurtosistech/files-artifacts-expander:${KURTOSIS_VERSION}" -o /tmp/docker-cache/kurtosis-expander.tar
          docker save "${KURTOSIS_VECTOR_IMAGE}" -o /tmp/docker-cache/vector.tar
          docker save "${KURTOSIS_FLUENTBIT_IMAGE}" -o /tmp/docker-cache/fluentbit.tar
          docker save "${KURTOSIS_CURL_JQ_IMAGE}" -o /tmp/docker-cache/curl-jq.tar
          docker save "${KURTOSIS_TRAEFIK_IMAGE}" -o /tmp/docker-cache/traefik.tar
          docker save "${KURTOSIS_ALPINE_IMAGE}" -o /tmp/docker-cache/alpine.tar
          docker save "${GENESIS_GENERATOR_IMAGE}" -o /tmp/docker-cache/genesis-generator.tar
          docker save "${ETH2_VAL_TOOLS_IMAGE}" -o /tmp/docker-cache/eth2-val-tools.tar

      - name: Save third-party containers to cache
        if: steps.cache-cl-images.outputs.cache-hit != 'true' && github.event_name != 'pull_request'
        uses: actions/cache/save@v6
        with:
          path: /tmp/docker-cache
          key: docker-cl-${{ env.ASSERTOOR_IMAGE }}-kurtosis-${{ env.KURTOSIS_VERSION }}-${{ env.KURTOSIS_VECTOR_IMAGE }}-${{ env.KURTOSIS_FLUENTBIT_IMAGE }}-${{ env.KURTOSIS_CURL_JQ_IMAGE }}-${{ env.KURTOSIS_TRAEFIK_IMAGE }}-${{ env.KURTOSIS_ALPINE_IMAGE }}-${{ env.GENESIS_GENERATOR_IMAGE }}-${{ env.ETH2_VAL_TOOLS_IMAGE }}
      - name: Install Kurtosis CLI and start engine
        uses: ./.github/actions/setup-kurtosis
        with:
          version: ${{ env.KURTOSIS_VERSION }}

      - name: Build erigon Docker image (with BuildKit layer cache)
        id: build_erigon_image
        continue-on-error: true
        uses: docker/build-push-action@v6
        with:
          context: .
          load: true
          tags: test/erigon:current
          build-args: |
            BINARIES=erigon caplin
          cache-from: type=gha,scope=kurtosis-erigon-build
          cache-to: type=gha,mode=max,scope=kurtosis-erigon-build

      # docker/build-push-action has no retry input; retry once so a transient
      # Docker Hub / GHA-cache failure doesn't fail an otherwise-good build.
      - name: Retry erigon Docker image build on transient failure
        if: steps.build_erigon_image.outcome == 'failure'
        uses: docker/build-push-action@v6
        with:
          context: .
          load: true
          tags: test/erigon:current
          build-args: |
            BINARIES=erigon caplin
          cache-from: type=gha,scope=kurtosis-erigon-build
          cache-to: type=gha,mode=max,scope=kurtosis-erigon-build

      - name: Run ${{ matrix.suite }} Kurtosis + assertoor tests
        timeout-minutes: ${{ matrix.test_timeout_minutes }}
        uses: erigontech/kurtosis-assertoor-github-action@v1.1.7
        with:
            enclave_name: "kurtosis-${{ matrix.suite }}-${{ github.run_id }}"
            ethereum_package_url: "github.com/ethpandaops/ethereum-package"
            ethereum_package_args: "${{ matrix.package_args }}"
            ethereum_package_branch: "${{ matrix.ethereum_package_branch }}"
            kurtosis_extra_args: --verbosity detailed --cli-log-level trace
            persistent_logs: "true"

      - name: Dump Kurtosis enclave on failure
        if: failure()
        continue-on-error: true
        run: |
          set -eu
          enclave_name="kurtosis-${{ matrix.suite }}-${{ github.run_id }}"
          dump_dir="${RUNNER_TEMP}/kurtosis-dump-${{ matrix.suite }}"
          kurtosis enclave dump "$enclave_name" "$dump_dir" || true

      - name: Upload Kurtosis enclave dump
        if: failure()
        continue-on-error: true
        uses: actions/upload-artifact@v7
        with:
          name: kurtosis-dump-${{ matrix.suite }}-${{ github.run_id }}
          path: ${{ runner.temp }}/kurtosis-dump-${{ matrix.suite }}
          retention-days: 7
          if-no-files-found: warn