[CI] Improve reliability of queries against GitHub GraphQL API (llvm#683)

Recently, the workflow for collecting llvm operational data has been
failing relatively often due to calls to the GitHub GraphQL API failing.
There doesn't seem to be definitive reason, with most responses just
being `502 Bad Gateway`. However, reducing the number of commits
requested per batch seems to make these calls reliable again.

This change also adds retries with some backoff, so the script only
fails after trying the GitHub API multiple times. We could also reduce
the number of commits we're requesting for each API failure, but just
reducing the overall batch size seems to work fine for the time being.

Author: jriv01

premerge/ops-container/process_llvm_commits.py

@@ -7,6 +7,7 @@
 import git
 from google.cloud import bigquery
 import requests
+import retry
 
 GITHUB_GRAPHQL_API_URL = "https://api.github.com/graphql"
 REPOSITORY_URL = "https://github.com/llvm/llvm-project.git"
@@ -17,7 +18,7 @@
 
 # How many commits to query the GitHub GraphQL API for at a time.
 # Querying too many commits at once often leads to the call failing.
-GITHUB_API_BATCH_SIZE = 50
+GITHUB_API_BATCH_SIZE = 35
 
 # Number of days to look back for new commits
 # We allow some buffer time between when a commit is made and when it is queried
@@ -71,6 +72,10 @@ class LLVMCommitInfo:
   commit_reverted: str | None = None
 
 
+class GitHubAPIError(Exception):
+  """Raised when a GitHub GraphQL API call fails."""
+
+
 def scrape_new_commits_by_date(
     target_datetime: datetime.datetime,
 ) -> list[git.Commit]:
@@ -105,6 +110,46 @@ def scrape_new_commits_by_date(
   return new_commits
 
 
+@retry.retry(
+    exceptions=(GitHubAPIError, requests.exceptions.ChunkedEncodingError),
+    tries=5,
+    delay=1,
+    backoff=2,
+)
+def query_github_graphql_api(
+    query: str,
+    github_token: str,
+) -> requests.Response:
+  """Query GitHub GraphQL API, retrying on failure.
+  
+  Args:
+    query: The GraphQL query to send to the GitHub API.
+    github_token: The access token to use with the GitHub GraphQL API.
+
+  Returns:
+    The response from the GitHub GraphQL API.
+  """
+  response = requests.post(
+      url=GITHUB_GRAPHQL_API_URL,
+      headers={
+          "Authorization": f"bearer {github_token}",
+      },
+      json={"query": query},
+  )
+
+  # Exit if API call fails
+  # A failed API call means a large batch of data is missing and will not be
+  # reflected in the dashboard. The dashboard will silently misrepresent
+  # commit data if we continue execution, so it's better to fail loudly.
+  if response.status_code < 200 or response.status_code >= 300:
+    raise GitHubAPIError(
+        "[%d] Failed to query GitHub GraphQL API: %s"
+        % (response.status_code, response.text)
+    )
+
+  return response
+
+
 def query_for_reviews(
     new_commits: list[git.Commit], github_token: str
 ) -> list[LLVMCommitInfo]:
@@ -189,21 +234,7 @@ def query_for_reviews(
         num_batches,
         len(subquery_batch),
     )
-    response = requests.post(
-        url=GITHUB_GRAPHQL_API_URL,
-        headers={
-            "Authorization": f"bearer {github_token}",
-        },
-        json={"query": query},
-    )
-
-    # Exit if API call fails
-    # A failed API call means a large batch of data is missing and will not be
-    # reflected in the dashboard. The dashboard will silently misrepresent
-    # commit data if we continue execution, so it's better to fail loudly.
-    if response.status_code < 200 or response.status_code >= 300:
-      logging.error("Failed to query GitHub GraphQL API: %s", response.text)
-      exit(1)
+    response = query_github_graphql_api(query, github_token)
 
     api_commit_data.update(response.json()["data"]["repository"])
 

premerge/ops-container/requirements.lock.txt

@@ -112,6 +112,12 @@ charset-normalizer==3.4.2 \
     # via
     #   -r ./requirements.txt
     #   requests
+decorator==5.2.1 \
+    --hash=sha256:65f266143752f734b0a7cc83c46f4618af75b8c5911b00ccb61d0ac9b6da0360 \
+    --hash=sha256:d316bb415a2d9e2d2b3abcc4084c6502fc09240e292cd76a76afc106a1c8e04a
+    # via
+    #   -r ./requirements.txt
+    #   retry
 gitdb==4.0.12 \
     --hash=sha256:5ef71f855d191a3326fcfbc0d5da835f26b13fbcba60c32c21091c349ffdb571 \
     --hash=sha256:67073e15955400952c6565cc3e707c554a4eea2e428946f7a4c162fab9bd9bcf
@@ -294,6 +300,12 @@ protobuf==6.31.1 \
     #   googleapis-common-protos
     #   grpcio-status
     #   proto-plus
+py==1.11.0 \
+    --hash=sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719 \
+    --hash=sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378
+    # via
+    #   -r ./requirements.txt
+    #   retry
 pyasn1==0.6.1 \
     --hash=sha256:0d632f46f2ba09143da3a8afe9e33fb6f92fa2320ab7e886e2d0f7672af84629 \
     --hash=sha256:6f580d2bdd84365380830acf45550f2511469f673cb4a5ae3857a3170128b034
@@ -320,6 +332,10 @@ requests==2.32.4 \
     #   -r ./requirements.txt
     #   google-api-core
     #   google-cloud-bigquery
+retry==0.9.2 \
+    --hash=sha256:ccddf89761fa2c726ab29391837d4327f819ea14d244c232a1d24c67a2f98606 \
+    --hash=sha256:f8bfa8b99b69c4506d6f5bd3b0aabf77f98cdb17f3c9fc3f5ca820033336fba4
+    # via -r ./requirements.txt
 rsa==4.9.1 \
     --hash=sha256:68635866661c6836b8d39430f97a996acbd61bfa49406748ea243539fe239762 \
     --hash=sha256:e7bdbfdb5497da4c07dfd35530e1a902659db6ff241e39d9953cad06ebd0ae75

premerge/ops-container/requirements.txt

@@ -1,6 +1,7 @@
 cachetools==5.5.2
 certifi==2025.6.15
 charset-normalizer==3.4.2
+decorator==5.2.1
 gitdb==4.0.12
 GitPython==3.1.44
 google-api-core==2.25.1
@@ -16,10 +17,12 @@ idna==3.10
 packaging==25.0
 proto-plus==1.26.1
 protobuf==6.31.1
+py==1.11.0
 pyasn1==0.6.1
 pyasn1_modules==0.4.2
 python-dateutil==2.9.0.post0
 requests==2.32.4
+retry==0.9.2
 rsa==4.9.1
 six==1.17.0
 smmap==5.0.2