e2e.yml

name: e2e
on:
  push:
    branches: [ main, 'hotfix/**' ]
  pull_request:
  workflow_dispatch:
    inputs:
      pr_number:
        description: Pull request number to test
        required: true
      head_sha:
        description: Expected PR head SHA
        required: true
      image_tag:
        description: GHCR image tag for this trusted run
        required: true
  schedule:
    # Flake gap-filler, every 6h at :23 (off the congested :00/:30 scheduler slots).
    # Each tick runs only the lightweight schedule-gate; the expensive setup + e2e
    # matrix run only when the gate returns proceed=true (no merge to main in
    # STALE_HOURS AND e2e idle AND toggle enabled). The 6h cadence bounds how often
    # it can fire during a long no-merge lull. See schedule-gate below and
    # malbeclabs/infra docs/plans/2026-07-02-e2e-flake-gap-filler-*.
    - cron: '23 */6 * * *'

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}

env:
  SHARD_COUNT: "5"
  # Use registry-qualified image names so we can push/pull between jobs
  DZ_IMAGE_REPO: ghcr.io/malbeclabs/dz-e2e
  DZ_IMAGE_TAG: ${{ github.event.inputs.image_tag || github.sha }}
  # Individual image refs for the e2e tests
  DZ_BASE_IMAGE: ghcr.io/malbeclabs/dz-e2e/base:${{ github.event.inputs.image_tag || github.sha }}
  DZ_LEDGER_IMAGE: ghcr.io/malbeclabs/dz-e2e/ledger:${{ github.event.inputs.image_tag || github.sha }}
  DZ_CONTROLLER_IMAGE: ghcr.io/malbeclabs/dz-e2e/controller:${{ github.event.inputs.image_tag || github.sha }}
  DZ_MANAGER_IMAGE: ghcr.io/malbeclabs/dz-e2e/manager:${{ github.event.inputs.image_tag || github.sha }}
  DZ_FUNDER_IMAGE: ghcr.io/malbeclabs/dz-e2e/funder:${{ github.event.inputs.image_tag || github.sha }}
  DZ_DEVICE_IMAGE: ghcr.io/malbeclabs/dz-e2e/device:${{ github.event.inputs.image_tag || github.sha }}
  DZ_CLIENT_IMAGE: ghcr.io/malbeclabs/dz-e2e/client:${{ github.event.inputs.image_tag || github.sha }}
  DZ_DEVICE_HEALTH_ORACLE_IMAGE: ghcr.io/malbeclabs/dz-e2e/device-health-oracle:${{ github.event.inputs.image_tag || github.sha }}
  DZ_GEOPROBE_IMAGE: ghcr.io/malbeclabs/dz-e2e/geoprobe:${{ github.event.inputs.image_tag || github.sha }}
  DZ_SENTINEL_IMAGE: ghcr.io/malbeclabs/dz-e2e/sentinel:${{ github.event.inputs.image_tag || github.sha }}
  DZ_IP_VERIFIER_IMAGE: ghcr.io/malbeclabs/dz-e2e/ip-verifier:${{ github.event.inputs.image_tag || github.sha }}
  DZ_VALIDATOR_METADATA_SERVICE_MOCK_IMAGE: ghcr.io/malbeclabs/dz-e2e/validator-metadata-service-mock:${{ github.event.inputs.image_tag || github.sha }}

jobs:
  schedule-gate:
    # Flake gap-filler gate. Runs on EVERY event so it is never skipped: a skipped
    # job would transitively skip the downstream e2e matrix (whose `if` has no
    # status-check function). On non-schedule events the script short-circuits to
    # proceed=true so setup and the matrix run exactly as on main; only the hourly
    # schedule path does the real gating (enabled AND stale AND idle). GitHub-hosted
    # so the hourly no-op never consumes a self-hosted e2e slot.
    name: Flake gap-filler gate
    runs-on: ubuntu-latest
    permissions:
      actions: read
    outputs:
      proceed: ${{ steps.gate.outputs.proceed }}
    env:
      # Default-off master switch. Set repo variable E2E_GAP_FILLER_ENABLED='true'
      # to activate (Settings -> Secrets and variables -> Actions -> Variables).
      GAP_FILLER_ENABLED: ${{ vars.E2E_GAP_FILLER_ENABLED }}
    steps:
      - name: Decide whether to run the gap-filler
        id: gate
        uses: actions/github-script@v7
        with:
          script: |
            // Only the hourly schedule triggers gap-filler gating. For every other
            // event (push/PR/dispatch) proceed unconditionally so nothing downstream
            // is skipped and the e2e matrix expands exactly as on main.
            if (context.eventName !== 'schedule') {
              core.info(`event=${context.eventName}: not the scheduled gap-filler; proceeding`);
              core.setOutput('proceed', 'true');
              return;
            }
            const STALE_HOURS = 6;
            const enabled = process.env.GAP_FILLER_ENABLED === 'true';
            const selfId = context.runId;
            const { owner, repo } = context.repo;

            // Stale check: newest *merge* (push) e2e run on main. Only push events
            // count — the gap-filler's own scheduled ticks are also branch=main e2e
            // runs, so counting them would make main never look stale (each tick
            // sees the prior tick ~one cron-interval ago and declines forever). We
            // want "no real e2e ran on main in STALE_HOURS", i.e. no merge.
            // Missing => Infinity => stale.
            const mainRuns = await github.rest.actions.listWorkflowRuns({
              owner, repo, workflow_id: 'e2e.yml', branch: 'main', event: 'push', per_page: 30,
            });
            const newestMain = mainRuns.data.workflow_runs
              .filter(r => r.id !== selfId)
              .sort((a, b) => new Date(b.created_at) - new Date(a.created_at))[0];
            const ageHours = newestMain
              ? (Date.now() - new Date(newestMain.created_at).getTime()) / 3.6e6
              : Infinity;
            const stale = ageHours >= STALE_HOURS;

            // Idle check: any e2e run not yet completed, on ANY branch, excluding
            // this run. These GitHub run statuses precede 'completed'.
            const active = [];
            for (const status of ['queued', 'in_progress', 'requested', 'waiting', 'pending']) {
              const res = await github.rest.actions.listWorkflowRuns({
                owner, repo, workflow_id: 'e2e.yml', status, per_page: 100,
              });
              active.push(...res.data.workflow_runs.filter(r => r.id !== selfId));
            }
            const idle = active.length === 0;

            const proceed = enabled && stale && idle;
            const ageStr = Number.isFinite(ageHours) ? ageHours.toFixed(1) + 'h' : 'none';
            core.info(`enabled=${enabled} stale=${stale} (newest merge age=${ageStr}, threshold=${STALE_HOURS}h) idle=${idle} (active=${active.length}) => proceed=${proceed}`);
            core.setOutput('proceed', String(proceed));
            await core.summary.addRaw(
              `### e2e flake gap-filler gate\n` +
              `- enabled: **${enabled}**\n` +
              `- stale: **${stale}** (newest merge (push) run age ${ageStr} / ${STALE_HOURS}h)\n` +
              `- idle: **${idle}** (in-flight e2e runs excluding self: ${active.length})\n` +
              `- **proceed: ${proceed}**\n`
            ).write();

  setup:
    needs: [schedule-gate]
    # schedule-gate runs on every event and never skips, so no always() is needed
    # here and nothing downstream (the e2e matrix) is transitively skipped. On
    # non-schedule events proceed is always 'true'; on the hourly schedule it is
    # 'true' only when the gap-filler is enabled AND main is stale AND e2e is idle.
    if: needs.schedule-gate.outputs.proceed == 'true'
    runs-on: self-hosted
    permissions:
      packages: write
      contents: read
      checks: write
    outputs:
      run-e2e: ${{ steps.gate.outputs.run-e2e }}
      matrix: ${{ steps.shard.outputs.matrix }}
    steps:
      # Version-bump PRs (only version lines change in Cargo.toml/Cargo.lock)
      # and docs-only PRs (markdown plus rfcs images) get no signal from e2e. When
      # skipping, report the required shard checks as successful on the PR
      # head, because the gated matrix job never creates its check runs and
      # branch protection would otherwise block the merge. Keep in sync with
      # shreds-e2e.yml.
      - name: Check for docs/version-bump-only PR
        id: skip-check
        if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository
        uses: actions/github-script@v7
        env:
          CHECK_NAME: e2e
          # SHARD_COUNT round-robin shards + 1 dedicated shard; must match the
          # required status check contexts in the main ruleset.
          CHECK_SHARDS: "6"
        with:
          script: |
            try {
              const files = await github.paginate(github.rest.pulls.listFiles, {
                owner: context.repo.owner,
                repo: context.repo.repo,
                pull_number: context.payload.pull_request.number,
                per_page: 100,
              });
              // Missing patch (diff too large) fails closed: e2e runs.
              const versionLinesOnly = (patch) => typeof patch === 'string' && patch
                .split('\n')
                .filter((l) => /^[+-]/.test(l) && !/^(\+\+\+|---)/.test(l))
                .every((l) => /^[+-]version = "/.test(l));
              const isMd = (name) => typeof name === 'string' && name.endsWith('.md');
              // Images under rfcs/ are figures for markdown RFCs; inert for e2e.
              const isRfcImage = (name) => typeof name === 'string' &&
                name.startsWith('rfcs/') && /\.(png|jpe?g|gif|svg|webp)$/i.test(name);
              const isInert = (name) => isMd(name) || isRfcImage(name);
              // Markdown and rfcs images are inert for e2e; a rename/copy must
              // come from an inert path too, since a rename also deletes the
              // source path.
              const fileOk = (f) => {
                if (isInert(f.filename)) {
                  return (f.status !== 'renamed' && f.status !== 'copied') || isInert(f.previous_filename);
                }
                return (f.filename === 'Cargo.toml' || f.filename === 'Cargo.lock') &&
                  f.status === 'modified' &&
                  versionLinesOnly(f.patch);
              };
              const names = new Set(files.map((f) => f.filename));
              // Cargo.toml and Cargo.lock must change together: a requirement
              // change that leaves the lock untouched can still alter what
              // unlocked CI builds resolve, and is not a version bump.
              const skippable = files.length > 0 &&
                names.has('Cargo.toml') === names.has('Cargo.lock') &&
                files.every(fileOk);
              core.setOutput('skip', String(skippable));
              if (!skippable) return;
              core.notice('Docs/version-bump-only PR; skipping e2e and reporting shard checks as successful.');
              const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
              for (let shard = 1; shard <= Number(process.env.CHECK_SHARDS); shard++) {
                await github.rest.checks.create({
                  owner: context.repo.owner,
                  repo: context.repo.repo,
                  name: `${process.env.CHECK_NAME} (shard ${shard})`,
                  head_sha: context.payload.pull_request.head.sha,
                  status: 'completed',
                  conclusion: 'success',
                  details_url: runUrl,
                  output: {
                    title: 'Skipped: docs/version-bump-only PR',
                    summary: 'Only markdown files, rfcs images, and version lines in Cargo.toml/Cargo.lock changed, so e2e was skipped.',
                  },
                });
              }
            } catch (err) {
              core.warning(`Skip gate failed; running e2e: ${err}`);
              core.setOutput('skip', 'false');
            }
      - name: Decide whether privileged e2e can run
        id: gate
        run: |
          if [ "${{ steps.skip-check.outputs.skip }}" = "true" ]; then
            echo "run-e2e=false" >> "$GITHUB_OUTPUT"
            echo "::notice::Skipping e2e: docs/version-bump-only PR."
            exit 0
          fi
          if [ "${{ github.event_name }}" = "pull_request" ] && [ "${{ github.event.pull_request.user.login }}" = "dependabot[bot]" ]; then
            echo "run-e2e=false" >> "$GITHUB_OUTPUT"
            echo "::notice::Skipping privileged e2e on a Dependabot PR (read-only token, no secrets). A maintainer can comment /run-e2e to start a trusted run."
            {
              echo "### Privileged e2e skipped"
              echo
              echo "Dependabot PRs run with a read-only token and no secrets, so privileged e2e cannot run on the \`pull_request\` event."
              echo "A maintainer can comment \`/run-e2e\` to dispatch trusted e2e for this PR head SHA."
            } >> "$GITHUB_STEP_SUMMARY"
            exit 0
          fi
          if [ "${{ github.event_name }}" = "pull_request" ] && [ "${{ github.event.pull_request.head.repo.full_name }}" != "${{ github.repository }}" ]; then
            echo "run-e2e=false" >> "$GITHUB_OUTPUT"
            echo "::notice::Skipping privileged e2e on an external fork PR. A maintainer can comment /run-e2e to start a trusted run."
            {
              echo "### Privileged e2e skipped"
              echo
              echo "This pull request comes from an external fork, so GitHub only grants a read-only token and no secrets."
              echo "A maintainer can comment \`/run-e2e\` to dispatch trusted e2e for this PR head SHA."
            } >> "$GITHUB_STEP_SUMMARY"
            exit 0
          fi
          echo "run-e2e=true" >> "$GITHUB_OUTPUT"
      - uses: actions/checkout@v4
        if: steps.gate.outputs.run-e2e == 'true' && github.event_name != 'workflow_dispatch'
      - uses: actions/checkout@v4
        if: steps.gate.outputs.run-e2e == 'true' && github.event_name == 'workflow_dispatch'
        with:
          ref: refs/pull/${{ github.event.inputs.pr_number }}/merge
          fetch-depth: 2
      - name: Validate trusted PR head
        if: steps.gate.outputs.run-e2e == 'true' && github.event_name == 'workflow_dispatch'
        run: |
          actual_head="$(git rev-parse HEAD^2)"
          if [ "$actual_head" != "${{ github.event.inputs.head_sha }}" ]; then
            echo "::error::PR head SHA changed: expected ${{ github.event.inputs.head_sha }}, got $actual_head"
            exit 1
          fi
      - uses: actions/setup-go@v5
        if: steps.gate.outputs.run-e2e == 'true'
        with:
          go-version-file: go.mod
          cache: true
      - name: Login to GHCR
        if: steps.gate.outputs.run-e2e == 'true'
        uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
      - name: Install make
        if: steps.gate.outputs.run-e2e == 'true'
        run: sudo apt-get update && sudo apt-get install -y make
      - name: Build images
        if: steps.gate.outputs.run-e2e == 'true'
        working-directory: e2e/
        run: go run ./cmd/dzctl/main.go build
      - name: Push images to registry
        if: steps.gate.outputs.run-e2e == 'true'
        run: |
          docker push ${{ env.DZ_IMAGE_REPO }}/base:${{ env.DZ_IMAGE_TAG }}
          docker push ${{ env.DZ_IMAGE_REPO }}/ledger:${{ env.DZ_IMAGE_TAG }}
          docker push ${{ env.DZ_IMAGE_REPO }}/controller:${{ env.DZ_IMAGE_TAG }}
          docker push ${{ env.DZ_IMAGE_REPO }}/manager:${{ env.DZ_IMAGE_TAG }}
          docker push ${{ env.DZ_IMAGE_REPO }}/funder:${{ env.DZ_IMAGE_TAG }}
          docker push ${{ env.DZ_IMAGE_REPO }}/device:${{ env.DZ_IMAGE_TAG }}
          docker push ${{ env.DZ_IMAGE_REPO }}/client:${{ env.DZ_IMAGE_TAG }}
          docker push ${{ env.DZ_IMAGE_REPO }}/device-health-oracle:${{ env.DZ_IMAGE_TAG }}
          docker push ${{ env.DZ_IMAGE_REPO }}/geoprobe:${{ env.DZ_IMAGE_TAG }}
          docker push ${{ env.DZ_IMAGE_REPO }}/sentinel:${{ env.DZ_IMAGE_TAG }}
          docker push ${{ env.DZ_IMAGE_REPO }}/ip-verifier:${{ env.DZ_IMAGE_TAG }}
          docker push ${{ env.DZ_IMAGE_REPO }}/validator-metadata-service-mock:${{ env.DZ_IMAGE_TAG }}
      - name: Discover tests and distribute across shards
        if: steps.gate.outputs.run-e2e == 'true'
        id: shard
        working-directory: e2e/
        run: |
          # Find files with exactly the e2e build tag (excludes compound tags like "e2e && stress")
          tests=$(grep -rl '^//go:build e2e$' ./*_test.go \
            | xargs grep -h '^func TestE2E_' \
            | sed 's/func \(TestE2E_[a-zA-Z0-9_]*\).*/\1/' \
            | sort)

          count=$(echo "$tests" | wc -l)
          echo "Discovered $count tests"
          echo "$tests"

          # Shard 1: BackwardCompatibility gets its own shard because it spawns
          # many parallel subtests internally and would starve other tests.
          dedicated="TestE2E_BackwardCompatibility"

          remaining=$(echo "$tests" | grep -v "^${dedicated}$")

          # Distribute remaining tests round-robin across shards
          declare -a shards
          for ((i=0; i<SHARD_COUNT; i++)); do
            shards[i]=""
          done

          i=0
          while IFS= read -r test; do
            idx=$((i % SHARD_COUNT))
            if [ -n "${shards[idx]}" ]; then
              shards[idx]="${shards[idx]}|${test}"
            else
              shards[idx]="$test"
            fi
            i=$((i + 1))
          done <<< "$remaining"

          # Build JSON matrix: shard 1 = BackwardCompatibility, shards 2+ = round-robin
          matrix="[{\"shard\":1,\"run\":\"^(${dedicated})$\"}"
          for ((i=0; i<SHARD_COUNT; i++)); do
            matrix="${matrix},{\"shard\":$((i + 2)),\"run\":\"^(${shards[i]})$\"}"
          done
          matrix="${matrix}]"

          echo "Matrix: $matrix"
          echo "matrix=$matrix" >> "$GITHUB_OUTPUT"

  e2e:
    name: e2e (shard ${{ matrix.shard }})
    needs: setup
    if: needs.setup.outputs.run-e2e == 'true'
    strategy:
      fail-fast: false
      matrix:
        include: ${{ fromJSON(needs.setup.outputs.matrix) }}
    defaults:
      run:
        working-directory: "e2e/"
    runs-on: doublezero-k8s-ci
    timeout-minutes: 15
    permissions:
      packages: read
      contents: read
      checks: write
    steps:
      - uses: actions/checkout@v4
        if: github.event_name != 'workflow_dispatch'
      - uses: actions/checkout@v4
        if: github.event_name == 'workflow_dispatch'
        with:
          ref: refs/pull/${{ github.event.inputs.pr_number }}/merge
          fetch-depth: 2
      - name: Validate trusted PR head
        if: github.event_name == 'workflow_dispatch'
        run: |
          actual_head="$(git rev-parse HEAD^2)"
          if [ "$actual_head" != "${{ github.event.inputs.head_sha }}" ]; then
            echo "::error::PR head SHA changed: expected ${{ github.event.inputs.head_sha }}, got $actual_head"
            exit 1
          fi
      # Trusted (workflow_dispatch) runs execute on the base ref, so their native
      # check runs attach to that commit, not the PR head. Report a check run on
      # the validated PR head SHA so branch protection's required context is met.
      - name: Report check-run start on PR head
        if: github.event_name == 'workflow_dispatch'
        id: check
        uses: actions/github-script@v7
        with:
          script: |
            const { data } = await github.rest.checks.create({
              owner: context.repo.owner,
              repo: context.repo.repo,
              name: 'e2e (shard ${{ matrix.shard }})',
              head_sha: context.payload.inputs.head_sha,
              status: 'in_progress',
              details_url: `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`,
            });
            core.setOutput('id', String(data.id));
      - uses: actions/setup-go@v5
        with:
          go-version-file: go.mod
          cache: true
      - name: Login to GHCR
        uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
      - name: Login to Docker Hub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}
      - name: Pull pre-built images
        run: |
          pull_with_retry() {
            local image=$1
            local max_attempts=3
            local attempt=1
            while [ $attempt -le $max_attempts ]; do
              echo "Pulling $image (attempt $attempt/$max_attempts)"
              if docker pull "$image"; then
                return 0
              fi
              echo "Pull failed, retrying in 5 seconds..."
              sleep 5
              attempt=$((attempt + 1))
            done
            echo "Failed to pull $image after $max_attempts attempts"
            return 1
          }
          pull_with_retry ${{ env.DZ_IMAGE_REPO }}/base:${{ env.DZ_IMAGE_TAG }}
          pull_with_retry ${{ env.DZ_IMAGE_REPO }}/ledger:${{ env.DZ_IMAGE_TAG }}
          pull_with_retry ${{ env.DZ_IMAGE_REPO }}/controller:${{ env.DZ_IMAGE_TAG }}
          pull_with_retry ${{ env.DZ_IMAGE_REPO }}/manager:${{ env.DZ_IMAGE_TAG }}
          pull_with_retry ${{ env.DZ_IMAGE_REPO }}/funder:${{ env.DZ_IMAGE_TAG }}
          pull_with_retry ${{ env.DZ_IMAGE_REPO }}/device:${{ env.DZ_IMAGE_TAG }}
          pull_with_retry ${{ env.DZ_IMAGE_REPO }}/client:${{ env.DZ_IMAGE_TAG }}
          pull_with_retry ${{ env.DZ_IMAGE_REPO }}/device-health-oracle:${{ env.DZ_IMAGE_TAG }}
          pull_with_retry ${{ env.DZ_IMAGE_REPO }}/geoprobe:${{ env.DZ_IMAGE_TAG }}
          pull_with_retry ${{ env.DZ_IMAGE_REPO }}/sentinel:${{ env.DZ_IMAGE_TAG }}
          pull_with_retry ${{ env.DZ_IMAGE_REPO }}/ip-verifier:${{ env.DZ_IMAGE_TAG }}
          pull_with_retry ${{ env.DZ_IMAGE_REPO }}/validator-metadata-service-mock:${{ env.DZ_IMAGE_TAG }}
          pull_with_retry ghcr.io/malbeclabs/dz-e2e/prometheus:v2.54.1
          pull_with_retry public.ecr.aws/influxdb/influxdb:1.8
      - name: test
        id: test
        env:
          DZ_E2E_NO_BUILD: "1"
          # TestE2E_BackwardCompatibility asks the GitHub API which client
          # releases exist. Unauthenticated, that is 60 requests an hour shared
          # across every runner on the same address, so the shard failed on a
          # 403 rate limit before it reached a devnet. The token raises the
          # limit to 1,000 an hour for this repository.
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: go test -tags=e2e -timeout=20m -parallel=12 -run '${{ matrix.run }}' -v
      - name: Report check-run result on PR head
        if: always() && github.event_name == 'workflow_dispatch' && steps.check.outputs.id
        uses: actions/github-script@v7
        with:
          script: |
            const outcome = '${{ steps.test.outcome }}';
            const conclusion = outcome === 'success' ? 'success' : (outcome === 'cancelled' ? 'cancelled' : 'failure');
            await github.rest.checks.update({
              owner: context.repo.owner,
              repo: context.repo.repo,
              check_run_id: Number('${{ steps.check.outputs.id }}'),
              status: 'completed',
              conclusion,
              details_url: `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`,
            });