exponential backoff for urlopen

Author: AchimGaedkeLynker

recipe/fixup_sha.py

@@ -1,21 +1,48 @@
 # Call like python3 __file__ python/ray/_version.py 2.49.2
 
-import json; 
+import json
 import os
 import sys
-from urllib.request import urlopen; 
+import time
+from urllib.request import HTTPError, urlopen
+
+
+def get_commit_sha(ver: str) -> str:
+    """
+    Return sha from version tag commit
+    
+    Use exponential backoff to address rate limiting issues.
+    """
+    timeout = 16
+    while True:
+        try:
+            content = urlopen(
+                f"https://api.github.com/repos/ray-project/ray/git/ref/tags/ray-{ver}"
+            ).read()
+            return json.loads(content)["object"]["sha"]
+        except HTTPError as e:
+            if e.code == 403 and e.msg == "rate limit exceeded" and timeout <= 128:
+                print(f"{e}, retrying after {timeout} s", file=sys.stderr)
+                print("headers", e.hdrs, file=sys.stderr)
+                print("env keys", list(os.environ))
+                time.sleep(timeout)
+                timeout *= 2
+                continue
+            raise
+
 
 versionpy = sys.argv[1]
 ver = sys.argv[2]
 
 assert os.path.exists(versionpy)
 
-content = urlopen(f'https://api.github.com/repos/ray-project/ray/git/ref/tags/ray-{ver}').read()
-sha = json.loads(content)['object']['sha']
-with open(versionpy, 'r') as fid:
+with open(versionpy, "r") as fid:
     txt = fid.read()
+
+sha = get_commit_sha(ver)
+
 txt = txt.replace("{{RAY_COMMIT_SHA}}", sha)
 assert f'version = "{ver}"' in txt
 assert f'commit = "{sha}"' in txt
-with open(versionpy, 'w') as fid:
+with open(versionpy, "w") as fid:
     fid.write(txt)