Retry uncached CPU test image builds

Signed-off-by: tamohannes <hovhannes.tamoyan@gmail.com>

Author: tamohannes

.github/workflows/tests.yml

@@ -9,7 +9,6 @@ on:
 
 permissions:
   contents: read
-  packages: read
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
@@ -18,18 +17,23 @@ concurrency:
 jobs:
   unit-tests:
     runs-on: ubuntu-latest
+    env:
+      HAS_GHCR_PAT: ${{ secrets.GHCR_PAT != '' }}
     steps:
     - name: Checkout
       uses: actions/checkout@v6
     - name: Set up Docker Buildx
       uses: docker/setup-buildx-action@v3
 
     - name: Login to GHCR
+      if: ${{ env.HAS_GHCR_PAT == 'true' }}
       uses: docker/login-action@v3
+      env:
+        GHCR_PAT: ${{ secrets.GHCR_PAT }}
       with:
         registry: ghcr.io
         username: ${{ github.actor }}
-        password: ${{ secrets.GHCR_PAT || github.token }}
+        password: ${{ env.GHCR_PAT }}
 
     - name: Free up disk space on Ubuntu
       run: |
@@ -55,17 +59,32 @@ jobs:
         # Calculate image tags that match what Python code expects
         # Python generates: locally-built-{sanitized_path}:{sha256_hash[:12]}
         REPO_LOWER=$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]')
-        CACHE_ARGS_NEMO=(--cache-from type=registry,ref=ghcr.io/${REPO_LOWER}/nemo-skills-image:cache)
-        CACHE_ARGS_SANDBOX=(--cache-from type=registry,ref=ghcr.io/${REPO_LOWER}/nemo-skills-sandbox-image:cache)
+        CACHE_ARGS_NEMO=()
+        CACHE_ARGS_SANDBOX=()
         if [ -n "${GHCR_PAT:-}" ]; then
+          CACHE_ARGS_NEMO+=(--cache-from type=registry,ref=ghcr.io/${REPO_LOWER}/nemo-skills-image:cache)
           CACHE_ARGS_NEMO+=(--cache-to type=registry,ref=ghcr.io/${REPO_LOWER}/nemo-skills-image:cache,mode=min)
+          CACHE_ARGS_SANDBOX+=(--cache-from type=registry,ref=ghcr.io/${REPO_LOWER}/nemo-skills-sandbox-image:cache)
           CACHE_ARGS_SANDBOX+=(--cache-to type=registry,ref=ghcr.io/${REPO_LOWER}/nemo-skills-sandbox-image:cache,mode=min)
         fi
 
+        retry_docker_build() {
+          for attempt in 1 2 3; do
+            if docker buildx build "$@"; then
+              return 0
+            fi
+            if [ "$attempt" -lt 3 ]; then
+              echo "Docker build failed on attempt ${attempt}; retrying in 30s..."
+              sleep 30
+            fi
+          done
+          return 1
+        }
+
         # Build nemo-skills-image with expected tag
         NEMO_SKILLS_HASH=$(sha256sum dockerfiles/Dockerfile.nemo-skills | cut -d' ' -f1 | cut -c1-12)
         NEMO_SKILLS_TAG="locally-built-dockerfiles-dockerfile-nemo-skills:${NEMO_SKILLS_HASH}"
-        docker buildx build \
+        retry_docker_build \
           --tag nemo-skills-image \
           --tag ${NEMO_SKILLS_TAG} \
           --file dockerfiles/Dockerfile.nemo-skills \
@@ -82,7 +101,7 @@ jobs:
         # Build sandbox-image with expected tag
         SANDBOX_HASH=$(sha256sum dockerfiles/Dockerfile.sandbox | cut -d' ' -f1 | cut -c1-12)
         SANDBOX_TAG="locally-built-dockerfiles-dockerfile-sandbox:${SANDBOX_HASH}"
-        docker buildx build \
+        retry_docker_build \
           --tag nemo-skills-sandbox-image \
           --tag ${SANDBOX_TAG} \
           --file dockerfiles/Dockerfile.sandbox \