fix: Docker push retries (ai-dynamo#6456)

Signed-off-by: Dmitry Tokarev <dtokarev@nvidia.com>

Author: dmitry-tokarev-nv

.github/actions/docker-tag-push/action.yml

@@ -51,17 +51,19 @@ runs:
         ECR_HOSTNAME: ${{ inputs.aws_account_id }}.dkr.ecr.${{ inputs.aws_default_region }}.amazonaws.com
       run: |
         set -euo pipefail
-        if [[ ${CONDITIONAL_TAG} != '' ]]; then
-            docker tag ${LOCAL_IMAGE} ${ECR_HOSTNAME}/${CONDITIONAL_TAG}
-            docker push ${ECR_HOSTNAME}/${CONDITIONAL_TAG}
+        source "${{ github.action_path }}/retry_push.sh"
+
+        if [[ -n "${CONDITIONAL_TAG}" ]]; then
+            docker tag "${LOCAL_IMAGE}" "${ECR_HOSTNAME}/${CONDITIONAL_TAG}"
+            retry_push "${ECR_HOSTNAME}/${CONDITIONAL_TAG}"
         fi
         while IFS= read -r TAG; do
           if [ -z "$TAG" ]; then
             continue
           fi
           echo "Tagging and pushing: ${ECR_HOSTNAME}/${TAG}"
           docker tag "${LOCAL_IMAGE}" "${ECR_HOSTNAME}/${TAG}"
-          docker push "${ECR_HOSTNAME}/${TAG}"
+          retry_push "${ECR_HOSTNAME}/${TAG}"
         done <<< "$PUSH_TAGS"
     - name: ACR Tag and Push
       shell: bash
@@ -72,11 +74,13 @@ runs:
         AZURE_ACR_HOSTNAME: ${{ inputs.azure_acr_hostname }}
       run: |
         set -euo pipefail
+        source "${{ github.action_path }}/retry_push.sh"
+
         while IFS= read -r TAG; do
           if [ -z "$TAG" ]; then
             continue
           fi
           echo "Tagging and pushing: ${AZURE_ACR_HOSTNAME}/${TAG}"
           docker tag "${LOCAL_IMAGE}" "${AZURE_ACR_HOSTNAME}/${TAG}"
-          docker push "${AZURE_ACR_HOSTNAME}/${TAG}"
+          retry_push "${AZURE_ACR_HOSTNAME}/${TAG}"
         done <<< "$PUSH_TAGS"

.github/actions/docker-tag-push/retry_push.sh

@@ -0,0 +1,29 @@
+# Retry docker push with exponential backoff.
+# Safe under `set -e`: the `if` conditional context prevents a failed
+# `docker push` from triggering an immediate exit.
+retry_push() {
+  local image="$1"
+  local max_attempts=3
+  local wait_seconds=10
+  local attempt=1
+
+  while true; do
+    if docker push "$image"; then
+      return 0
+    fi
+    echo "Push failed for $image (attempt ${attempt}/${max_attempts})." >&2
+
+    if (( attempt >= max_attempts )); then
+      echo "Push failed after ${max_attempts} attempts: $image" >&2
+      return 1
+    fi
+
+    echo "Retrying in ${wait_seconds}s..."
+    sleep "$wait_seconds"
+    attempt=$((attempt + 1))
+    wait_seconds=$((wait_seconds * 2))
+    if (( wait_seconds > 120 )); then
+      wait_seconds=120
+    fi
+  done
+}

.github/actions/skopeo-copy/action.yml

@@ -109,7 +109,7 @@ runs:
         RETRY_DELAY=10
         for attempt in $(seq 1 $MAX_RETRIES); do
           echo "Attempt ${attempt}/${MAX_RETRIES}..."
-          if skopeo copy --all "${SOURCE_REF}" "${TARGET_REF}"; then
+          if skopeo copy --all --retry-times 3 "${SOURCE_REF}" "${TARGET_REF}"; then
             echo "target_image_ref=${{ inputs.target_registry }}/${TARGET_IMAGE}:${TARGET_TAG}" >> $GITHUB_OUTPUT
             echo "✅ Image copied successfully"
             exit 0

.github/workflows/build-test-distribute-flavor.yml

@@ -90,7 +90,7 @@ on:
         description: 'Timeout in minutes for the copy to ACR step'
         required: false
         type: number
-        default: 5
+        default: 10
     secrets:
       AWS_DEFAULT_REGION:
         required: true