fix: pre-pull and cache k3s node image to prevent E2E cluster creation failures (#287)

k3d cluster creation fails intermittently when Docker Hub is slow or
rate-limited because the k3s node image (rancher/k3s) is not pre-pulled
or cached. While cert-manager, Prometheus, stress-ng, and busybox images
are all pre-pulled and cached as tarballs, the k3s image was left to k3d
to pull on the fly during cluster creation.

This caused all 3 retry attempts to fail with 'Client.Timeout exceeded
while awaiting headers' in the nightly E2E run for K8s v1.34
(run 26858361803), while v1.32, v1.33, and v1.35 succeeded on runners
with better connectivity.

Changes:
- Add k3s image to pull_and_save() in both e2e-nightly.yaml and the
  setup-e2e-cluster composite action
- Add /tmp/k3s.tar to the actions/cache path and include the k3s
  version in the cache key
- Add docker load before k3d cluster create so the image is available
  locally regardless of Docker Hub connectivity
- Add retry logic (3 attempts with 10s backoff) to pull_and_save()
  for all images, not just k3s

Signed-off-by: Sebastien Tardif <sebtardif@ncf.ca>

Author: SebTardif

.github/actions/setup-e2e-cluster/action.yaml

@@ -59,7 +59,8 @@ runs:
           /tmp/cert-manager-cainjector.tar
           /tmp/prometheus.tar
           /tmp/stress-ng.tar
-        key: e2e-images-${{ runner.os }}-cm${{ inputs.cert-manager-version }}-prom${{ inputs.prometheus-chart-version }}-stress0.20.01
+          /tmp/k3s.tar
+        key: e2e-images-${{ runner.os }}-cm${{ inputs.cert-manager-version }}-prom${{ inputs.prometheus-chart-version }}-stress0.20.01-k3s${{ inputs.k3s-image }}
 
     - uses: ./.github/actions/install-binary-tool
       with:
@@ -87,24 +88,39 @@ runs:
         docker builder prune -f || true
         docker system prune -f || true
 
-    - name: Pre-pull cert-manager and Prometheus images
+    - name: Pre-pull container images
       shell: bash -Eeuo pipefail -x {0}
       run: |
+        # Pull images with docker (parallel layer downloads) and save as
+        # tarballs for k3d import. Skip images already restored from cache.
+        # Retries handle transient Docker Hub connectivity issues.
         pull_and_save() {
           local image="$1" tarball="$2"
           [[ -f "$tarball" ]] && { echo "Cached: $tarball"; return 0; }
-          docker pull --platform linux/amd64 "$image"
-          docker save "$image" -o "$tarball"
+          for attempt in 1 2 3; do
+            if docker pull --platform linux/amd64 "$image"; then
+              docker save "$image" -o "$tarball"
+              return 0
+            fi
+            echo "::warning::docker pull attempt $attempt for $image failed, retrying in 10s..."
+            sleep 10
+          done
+          echo "::error::Failed to pull $image after 3 attempts"
+          return 1
         }
         pull_and_save "quay.io/jetstack/cert-manager-controller:${{ inputs.cert-manager-version }}" /tmp/cert-manager-controller.tar
         pull_and_save "quay.io/jetstack/cert-manager-webhook:${{ inputs.cert-manager-version }}" /tmp/cert-manager-webhook.tar
         pull_and_save "quay.io/jetstack/cert-manager-cainjector:${{ inputs.cert-manager-version }}" /tmp/cert-manager-cainjector.tar
         pull_and_save "${{ inputs.prometheus-image }}" /tmp/prometheus.tar
         pull_and_save "${{ inputs.stress-ng-image }}" /tmp/stress-ng.tar
+        pull_and_save "${{ inputs.k3s-image }}" /tmp/k3s.tar
 
     - name: Create k3d cluster
       shell: bash -Eeuo pipefail -x {0}
       run: |
+        # Load the pre-pulled k3s image into Docker so k3d uses it
+        # without pulling from Docker Hub (avoids rate-limit/timeout failures).
+        docker load -i /tmp/k3s.tar
         for attempt in 1 2 3; do
           if k3d cluster create "${{ inputs.cluster-name }}" \
             --image ${{ inputs.k3s-image }} \

.github/workflows/e2e-nightly.yaml

@@ -135,7 +135,8 @@ jobs:
             /tmp/prometheus.tar
             /tmp/stress-ng.tar
             /tmp/busybox.tar
-          key: e2e-images-${{ runner.os }}-cm${{ env.CERT_MANAGER_VERSION }}-prom${{ env.PROMETHEUS_CHART_VERSION }}-stress0.20.01-busybox1.37
+            /tmp/k3s.tar
+          key: e2e-images-${{ runner.os }}-cm${{ env.CERT_MANAGER_VERSION }}-prom${{ env.PROMETHEUS_CHART_VERSION }}-stress0.20.01-busybox1.37-k3s${{ matrix.k3s-image }}
 
       - uses: ./.github/actions/install-binary-tool
         with:
@@ -163,28 +164,41 @@ jobs:
           docker builder prune -f || true
           docker system prune -f || true
 
-      - name: Pre-pull cert-manager and Prometheus images
+      - name: Pre-pull container images
         shell: bash -Eeuo pipefail -x {0}
         run: |
           # Pull images with docker (parallel layer downloads) and save as
           # tarballs for k3d import. Skip images already restored from cache.
+          # Retries handle transient Docker Hub connectivity issues.
           pull_and_save() {
             local image="$1" tarball="$2"
             [[ -f "$tarball" ]] && { echo "Cached: $tarball"; return 0; }
-            docker pull --platform linux/amd64 "$image"
-            docker save "$image" -o "$tarball"
+            for attempt in 1 2 3; do
+              if docker pull --platform linux/amd64 "$image"; then
+                docker save "$image" -o "$tarball"
+                return 0
+              fi
+              echo "::warning::docker pull attempt $attempt for $image failed, retrying in 10s..."
+              sleep 10
+            done
+            echo "::error::Failed to pull $image after 3 attempts"
+            return 1
           }
           pull_and_save "quay.io/jetstack/cert-manager-controller:${{ env.CERT_MANAGER_VERSION }}" /tmp/cert-manager-controller.tar
           pull_and_save "quay.io/jetstack/cert-manager-webhook:${{ env.CERT_MANAGER_VERSION }}" /tmp/cert-manager-webhook.tar
           pull_and_save "quay.io/jetstack/cert-manager-cainjector:${{ env.CERT_MANAGER_VERSION }}" /tmp/cert-manager-cainjector.tar
           pull_and_save "${{ env.PROMETHEUS_IMAGE }}" /tmp/prometheus.tar
           pull_and_save "${{ env.STRESS_NG_IMAGE }}" /tmp/stress-ng.tar
           pull_and_save "${{ env.CPU_BURN_IMAGE }}" /tmp/busybox.tar
+          pull_and_save "rancher/k3s:${{ matrix.k3s-image }}" /tmp/k3s.tar
 
       - name: Create k3d cluster
         shell: bash -Eeuo pipefail -x {0}
         run: |
           rm -f "$KUBECONFIG"
+          # Load the pre-pulled k3s image into Docker so k3d uses it
+          # without pulling from Docker Hub (avoids rate-limit/timeout failures).
+          docker load -i /tmp/k3s.tar
           # Retry cluster creation up to 3 times. k3d can fail transiently
           # when Docker containers vanish mid-creation due to daemon
           # contention with concurrent CI runs.