retry on merge failure

and just keep going when backfilling tags if a workflow fails

Author: joecorall

.github/workflows/build.yml

@@ -109,6 +109,26 @@ jobs:
         run: |
           set -eou pipefail
 
+          # Retry function with exponential backoff for rate limiting
+          retry_with_backoff() {
+            local max_attempts=5
+            local attempt=1
+            local delay=15
+            while true; do
+              if "$@"; then
+                return 0
+              fi
+              if [ $attempt -ge $max_attempts ]; then
+                echo "Command failed after $max_attempts attempts"
+                return 1
+              fi
+              echo "Attempt $attempt failed, retrying in ${delay}s..."
+              sleep $delay
+              attempt=$((attempt + 1))
+              delay=$((delay * 2))
+            done
+          }
+
           REPOS=(
             "${{ inputs.repository }}"
             "ghcr.io/islandora-devops"
@@ -131,7 +151,7 @@ jobs:
           FIRST_TAG=$(echo "${{ inputs.tags }}" | awk '{print $1}')
           BASE_IMAGE_REF="${{ inputs.repository }}/${{ inputs.image }}:${FIRST_TAG}"
 
-          docker buildx imagetools create \
+          retry_with_backoff docker buildx imagetools create \
             ${TAG_ARGS} \
             "${BASE_IMAGE_REF}-amd64" \
             "${BASE_IMAGE_REF}-arm64"

ci/backfill-tags.sh

@@ -36,7 +36,6 @@ DOCKER_IMAGES=(
   homarus
   houdini
   hypercube
-  imagemagick
   java
   mariadb
   milliner
@@ -96,14 +95,17 @@ get_workflow_file() {
   fi
 }
 
-# Check if a Docker image exists
+# Check if a Docker image exists using Docker Hub API
 image_exists() {
   local image="$1"
   local tag="$2"
-  local full_image="${DOCKER_REPOSITORY}/${image}:${tag}"
+  local api_url="https://hub.docker.com/v2/repositories/${DOCKER_REPOSITORY}/${image}/tags/${tag}"
 
-  # Use docker manifest inspect to check if image exists (doesn't pull the image)
-  if docker manifest inspect "$full_image" &>/dev/null; then
+  # Use Docker Hub API to check if tag exists (more reliable than docker manifest inspect)
+  local http_code
+  http_code=$(curl -s -o /dev/null -w "%{http_code}" "$api_url")
+
+  if [[ "$http_code" == "200" ]]; then
     return 0
   else
     return 1
@@ -167,17 +169,11 @@ run_workflow() {
   while true; do
     status=$(gh run view "$run_id" --json status,conclusion --jq '.status')
     if [[ "$status" == "completed" ]]; then
-      conclusion=$(gh run view "$run_id" --json conclusion --jq '.conclusion')
-      if [[ "$conclusion" != "success" ]]; then
-        echo "  Workflow failed with conclusion: $conclusion"
-        exit 1
-      fi
       break
     fi
     printf "."
     sleep 30
   done
-  echo ""
 
   echo "Workflow completed for tag: $tag"
 }