Retry API Gateway force deployment on throttling

Author: owenisas

.github/workflows/deploy-backend-aws.yml

@@ -693,10 +693,35 @@ jobs:
             --query 'Stacks[0].Outputs[?OutputKey==`ApiEndpoint`].OutputValue' \
             --output text)
           REST_API_ID=$(echo "$ENDPOINT" | sed -E 's#https://([^.]+)\.execute-api\..*#\1#')
-          aws apigateway create-deployment \
-            --rest-api-id "$REST_API_ID" \
-            --stage-name "${{ env.ENVIRONMENT }}" \
-            --description "github-actions-${GITHUB_RUN_ID}-${GITHUB_SHA}"
+          for attempt in 1 2 3 4 5; do
+            set +e
+            OUTPUT=$(aws apigateway create-deployment \
+              --rest-api-id "$REST_API_ID" \
+              --stage-name "${{ env.ENVIRONMENT }}" \
+              --description "github-actions-${GITHUB_RUN_ID}-${GITHUB_SHA}" 2>&1)
+            STATUS=$?
+            set -e
+
+            echo "$OUTPUT"
+
+            if [ "$STATUS" -eq 0 ]; then
+              exit 0
+            fi
+
+            if ! echo "$OUTPUT" | grep -q "TooManyRequestsException"; then
+              echo "::error::API Gateway stage deployment failed with a non-retryable error"
+              exit "$STATUS"
+            fi
+
+            if [ "$attempt" -eq 5 ]; then
+              echo "::error::API Gateway stage deployment hit AWS rate limits after 5 attempts"
+              exit "$STATUS"
+            fi
+
+            SLEEP_SECONDS=$((attempt * 15))
+            echo "API Gateway rate limited create-deployment; retrying in ${SLEEP_SECONDS}s..."
+            sleep "$SLEEP_SECONDS"
+          done
 
       - name: Get API Endpoint
         id: get-endpoint