e2e-ci.yml

# Run E2E tests for Fleet standalone
name: E2E Fleet

on:
  workflow_dispatch:
  pull_request:
  push:
    branches:
    - 'release/*'

env:
  GOARCH: amd64
  CGO_ENABLED: 0
  SETUP_K3D_VERSION: 'v5.8.3'

jobs:
  e2e-fleet-test:
    runs-on: ubuntu-latest

    strategy:
      fail-fast: false
      matrix:
        k3s:
          # k3d version list k3s | sed 's/+/-/' | sort -h
          # https://hub.docker.com/r/rancher/k3s/tags
          - name: k3s-new
            version: v1.33.1-k3s1
          - name: k3s-old
            version: v1.30.9-k3s1
        test_type:
          - name: default
          - name: sharding
            shards: '[{"id":"shard0"},{"id":"shard1"},{"id":"shard2"}]'
          - name: sharded-metrics
            shards: '[{"id":"shard0"}]'
          - name: metrics
          - name: infra-setup-git
          - name: infra-setup-helm
          - name: infra-setup-oci
    name: e2e-fleet-test-${{ matrix.k3s.name }}-${{ matrix.test_type.name }}

    steps:
      -
        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5
        with:
          fetch-depth: 0
      -
        uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # v6
        with:
          go-version-file: 'go.mod'
          check-latest: true
      -
        name: Noisy Neighbors
        run: |
          #!/usr/bin/env bash
          set -euo pipefail

          DURATION="${DURATION:-10}"   # seconds to run the probe
          WORKERS="${WORKERS:-0}"      # default: use all CPUs; set to cap parallelism

          echo "=== Noisy neighbor probe ==="
          date
          uname -a || true

          CPUS="$(nproc || echo 1)"
          [[ "$WORKERS" -gt 0 ]] || WORKERS="$CPUS"
          echo "CPUs: $CPUS  | Workers: $WORKERS  | Duration: ${DURATION}s"

          # Resolve cgroup v2 cpu.stat for this process
          CGROUP_SUBPATH="$(awk -F: '$1=="0"{print $3}' /proc/self/cgroup || true)"
          CG_BASE="/sys/fs/cgroup${CGROUP_SUBPATH}"
          CPU_STAT_PATH="${CG_BASE}/cpu.stat"
          if [[ ! -f "$CPU_STAT_PATH" ]]; then
            # Fallbacks for odd layouts
            [[ -f "/sys/fs/cgroup/cpu.stat" ]] && CPU_STAT_PATH="/sys/fs/cgroup/cpu.stat"
          fi

          print_cpu_stat() {
            if [[ -f "$CPU_STAT_PATH" ]]; then
              echo "--- cgroup cpu.stat ($CPU_STAT_PATH) ---"
              cat "$CPU_STAT_PATH"
            else
              echo "--- cgroup cpu.stat not found ---"
            fi
          }

          print_psi() {
            for r in cpu io memory; do
              local p="/proc/pressure/$r"
              if [[ -f "$p" ]]; then
                echo "--- PSI $r ---"
                cat "$p"
              fi
            done
          }

          print_load() {
            echo "--- Load /proc/loadavg ---"
            cat /proc/loadavg
          }

          echo
          echo "== Baseline =="
          print_load
          print_psi
          print_cpu_stat

          echo
          echo "== Running CPU workload for ${DURATION}s with ${WORKERS} worker(s)... =="

          # Worker: CPU-bound hash of endless stream; avoids disk and is available on runners
          run_worker() {
            timeout "${DURATION}s" bash -c 'yes | md5sum > /dev/null'
          }

          PIDS=()
          for _ in $(seq 1 "$WORKERS"); do
            run_worker &
            PIDS+=("$!")
          done

          # While workers run, sample PSI every second (lightweight)
          SAMPLES=()
          for ((i=0; i<"${DURATION}"; i++)); do
            if [[ -f /proc/pressure/cpu ]]; then
              line=$(sed -n '1p' /proc/pressure/cpu)   # "some ..." line
              SAMPLES+=("$(date +%H:%M:%S) $line")
            fi
            sleep 1
          done

          # Wait for workers
          for p in "${PIDS[@]}"; do wait "$p" 2>/dev/null || true; done

          echo
          echo "== After workload =="
          print_load
          print_psi
          print_cpu_stat

          if [[ ${#SAMPLES[@]} -gt 0 ]]; then
            echo
            echo "== PSI cpu 'some' samples during run (1/sec) =="
            printf '%s\n' "${SAMPLES[@]}"
          fi

          echo
          echo "== Interpretation hints =="
          cat <<'HINTS'
          • If loadavg (first number) >> CPU count while running, the run queue is saturated. That can be your own parallelism, platform limits, or neighbors.
          • In PSI:
            - cpu: 'some' rising toward 100% means tasks frequently stalled on CPU; 'full' > 0% means the whole workload often couldn't run at all.
            - io/memory pressure > 0% implies stalls waiting on disk/cache/mem — noisy neighbors or shared resource contention.
          • In cpu.stat:
            - nr_throttled / throttled_usec increasing => cgroup CPU quota throttling (platform-imposed). That looks like "steal", but it's policy not hypervisor steal.
          • If PSI stays near zero, load ~= workers, and no throttling, the bottleneck is likely your code or chosen parallelism, not neighbors.
          HINTS

      -
        name: Install Ginkgo CLI
        run: go install github.com/onsi/ginkgo/v2/ginkgo
      -
        name: Determine cache key
        id: cache-key
        run: |
          DAY_OF_YEAR=$(date +%j)
          if [ $(($DAY_OF_YEAR % 28)) -eq 0 ]; then
            echo "value=$(date +%Y-%m-%d)" >> $GITHUB_OUTPUT
          else
            echo "value=latest" >> $GITHUB_OUTPUT
          fi
      -
        name: Cache crust-gather CLI
        id: cache-crust
        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
        with:
          path: ~/.local/bin/crust-gather
          key: ${{ runner.os }}-crust-gather-${{ steps.cache-key.outputs.value }}
          restore-keys: |
            ${{ runner.os }}-crust-gather-
      -
        name: Install crust-gather CLI
        run: |
          if [ "${{ steps.cache-crust.outputs.cache-hit }}" != "true" ]; then
            echo "Cache not found, downloading from source"
            mkdir -p ~/.local/bin
            if curl -sSfL https://github.com/crust-gather/crust-gather/raw/main/install.sh | sh -s -- --yes; then
              # Cache the binary for future runs
              if [ ! -f ~/.local/bin/crust-gather ]; then
                which crust-gather && cp $(which crust-gather) ~/.local/bin/
              fi
            else
              echo "Failed to download crust-gather"
              exit 1
            fi
          else
            echo "Using cached crust-gather CLI"
            chmod +x ~/.local/bin/crust-gather
            sudo ln -sf ~/.local/bin/crust-gather /usr/local/bin/
          fi
      -
        name: Build Fleet
        run: |
          ./.github/scripts/build-fleet-binaries.sh
          ./.github/scripts/build-fleet-images.sh
      -
        name: Build Infra Tool
        if: ${{ startsWith(matrix.test_type.name, 'infra-setup') }}
        run: |
          pushd e2e/testenv/infra
          go build
          popd
          cd e2e/assets/gitrepo
          # Buildkit needed here for proper here-document support
          DOCKER_BUILDKIT=1 docker build -f Dockerfile.gitserver -t nginx-git:test --build-arg="passwd=$(openssl passwd foo)" .
      -
        name: Install k3d
        run: curl --silent --fail https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | TAG=${{ env.SETUP_K3D_VERSION }} bash
      -
        name: Provision k3d Cluster
        run: |
          k3d cluster create upstream --wait \
            --agents 1 \
            --network "nw01" \
            --image docker.io/rancher/k3s:${{matrix.k3s.version}}
      -
        name: Import Images Into k3d
        run: |
          ./.github/scripts/k3d-import-retry.sh rancher/fleet:dev rancher/fleet-agent:dev nginx-git:test -c upstream
      -
        name: Deploy Fleet
        env:
          SHARDS: ${{ matrix.test_type.shards }}
        run: |
          ./.github/scripts/deploy-fleet.sh
      -
        name: E2E Tests
        if: ${{ matrix.test_type.name == 'default' }}
        env:
          FLEET_E2E_NS: fleet-local
        run: |
          ginkgo --github-output --trace --label-filter='!infra-setup && !sharding' e2e/single-cluster e2e/keep-resources
      -
        name: E2E Sharding Tests
        if: ${{ matrix.test_type.name == 'sharding' }}
        env:
          FLEET_E2E_NS: fleet-local
        run: |
          ginkgo --github-output --trace --label-filter='sharding' e2e/single-cluster
      -
        name: E2E Sharded Metrics
        if: ${{ matrix.test_type.name == 'sharded-metrics' }}
        env:
          FLEET_E2E_NS: fleet-local
        run: |
          SHARD=shard0 ginkgo --github-output --trace --label-filter='!oci-registry' e2e/metrics
      -
        name: E2E Metrics Tests
        if: ${{ matrix.test_type.name == 'metrics' }}
        env:
          FLEET_E2E_NS: fleet-local
        run: |
          ginkgo --github-output --trace --label-filter='!oci-registry' e2e/metrics
      -
        name: Create Zot certificates for OCI tests
        if: ${{ startsWith(matrix.test_type.name, 'infra-setup') }}
        run: |
          ./.github/scripts/create-zot-certs.sh "FleetCI-RootCA"
          ./.github/scripts/create-secrets.sh 'FleetCI-RootCA'
      -
        name: E2E Infra Tests - Git
        if: ${{ matrix.test_type.name == 'infra-setup-git' }}
        env:
          FLEET_E2E_NS: fleet-local
          # Git and OCI credentials are here used in a local, ephemeral environment. Leaks would be harmless.
          GIT_HTTP_USER: "fleet-ci"
          GIT_HTTP_PASSWORD: "foo"
        run: |
          export CI_OCI_CERTS_DIR="$(git rev-parse --show-toplevel)/FleetCI-RootCA"

          # Run tests requiring only the git server
          e2e/testenv/infra/infra setup --git-server=true
          ginkgo --github-output --trace --label-filter='infra-setup && !helm-registry && !oci-registry' e2e/single-cluster/

          e2e/testenv/infra/infra teardown
      -
        name: E2E Infra Tests - Helm
        if: ${{ matrix.test_type.name == 'infra-setup-helm' }}
        env:
          FLEET_E2E_NS: fleet-local
          # Git and OCI credentials are here used in a local, ephemeral environment. Leaks would be harmless.
          GIT_HTTP_USER: "fleet-ci"
          GIT_HTTP_PASSWORD: "foo"
        run: |
          export CI_OCI_CERTS_DIR="$(git rev-parse --show-toplevel)/FleetCI-RootCA"

          # Run tests requiring a Helm registry (and a git server)
          e2e/testenv/infra/infra setup --git-server=true
          e2e/testenv/infra/infra setup --helm-registry=true

          ginkgo --github-output --trace --label-filter='helm-registry' e2e/single-cluster

          e2e/testenv/infra/infra teardown
      -
        name: E2E Infra Tests - OCI
        if: ${{ matrix.test_type.name == 'infra-setup-oci' }}
        env:
          # Git and OCI credentials are here used in a local, ephemeral environment. Leaks would be harmless.
          GIT_HTTP_USER: "fleet-ci"
          GIT_HTTP_PASSWORD: "foo"
          FLEET_E2E_NS: fleet-local
          CI_OCI_USERNAME: "fleet-ci"
          CI_OCI_PASSWORD: "foo"
          CI_OCI_READER_USERNAME: "fleet-ci-reader"
          CI_OCI_READER_PASSWORD: "foo-reader"
          CI_OCI_NO_DELETER_USERNAME: "fleet-ci-no-deleter"
          CI_OCI_NO_DELETER_PASSWORD: "foo-no-deleter"
        run: |
          export CI_OCI_CERTS_DIR="$(git rev-parse --show-toplevel)/FleetCI-RootCA"

          e2e/testenv/infra/infra setup --git-server=true
          e2e/testenv/infra/infra setup --helm-registry=true
          e2e/testenv/infra/infra setup --oci-registry=true

          ginkgo --github-output --trace --label-filter='oci-registry' e2e/single-cluster
          ginkgo --github-output --trace --label-filter='oci-registry' e2e/metrics

          e2e/testenv/infra/infra teardown
      -
        name: Fleet Tests Requiring Github Secrets
        # These tests can't run for PRs, because PRs don't have access to the secrets
        if: >
          matrix.test_type.name == 'default' &&
          github.event_name != 'pull_request' &&
          github.repository == 'rancher/fleet'

        env:
          FLEET_E2E_NS: fleet-local
          GIT_REPO_URL: "git@github.com:fleetrepoci/test.git"
          GIT_REPO_HOST: "github.com"
          GIT_REPO_USER: "git"
          GIT_REPO_BRANCH: ${{ matrix.k3s.version }}
          GITHUB_PAT_TOKEN: ${{ secrets.CI_PRIVATE_REPO_PAT }}
          CI_OCI_USERNAME:  ${{ secrets.CI_OCI_USERNAME }}
          CI_OCI_PASSWORD:  ${{ secrets.CI_OCI_PASSWORD }}
          GITHUB_APP_ID:  ${{ secrets.CI_GITHUB_APP_ID }}
          GITHUB_APP_INSTALLATION_ID:  ${{ secrets.CI_GITHUB_APP_INSTALLATION_ID }}
          GITHUB_APP_PRIVATE_KEY: ${{ secrets.CI_GITHUB_APP_PRIVATE_KEY }}
        run: |
          export GIT_SSH_KEY="$GITHUB_WORKSPACE/id_ecdsa"
          export GIT_SSH_PUBKEY="$GITHUB_WORKSPACE/id_ecdsa.pub"
          echo "${{ secrets.CI_SSH_KEY }}" > "$GIT_SSH_KEY"
          echo "${{ secrets.CI_SSH_PUBKEY }}" > "$GIT_SSH_PUBKEY"

          ginkgo --github-output --trace e2e/require-secrets
      -
        name: Upload Logs
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
        if: failure()
        with:
          name: gha-fleet-e2e-logs-${{ github.sha }}-${{ matrix.k3s.version }}-${{ matrix.test_type.name }}-${{ github.run_id }}
          path: |
            tmp/upstream
          retention-days: 2