Refactor GitHub repos list to use async requests (#4383)

Reduce the time when fetching multiple pages by performing the requests
asynchronously, using up to `ASYNC_CONCURRENCY` concurrent requests.

Closes
DIAGNijmegen/rse-grand-challenge-admin#664

Author: jmsmkn

app/grandchallenge/github/views.py

@@ -1,9 +1,12 @@
+import asyncio
 import hashlib
 import hmac
 import json
 from secrets import compare_digest
 
+import httpx
 import requests
+from asgiref.sync import async_to_sync
 from dal_select2.views import Select2ListView
 from django.conf import settings
 from django.contrib.auth.decorators import login_required
@@ -17,6 +20,7 @@
 from guardian.mixins import LoginRequiredMixin
 from requests import HTTPError
 
+from grandchallenge.components.backends.base import ASYNC_CONCURRENCY
 from grandchallenge.github.exceptions import GitHubBadRefreshTokenException
 from grandchallenge.github.models import GitHubUserToken, GitHubWebhookMessage
 from grandchallenge.github.utils import (
@@ -116,20 +120,18 @@ def github_app_install_url(self):
         return f"{settings.GITHUB_APP_INSTALL_URL}?state={self.github_state}"
 
     @cached_property
-    def github_request_kwargs(self):
+    def github_request_headers(self):
         return {
-            "headers": {
-                "Accept": "application/vnd.github+json",
-                "Authorization": f"token {self.github_user_token.access_token}",
-            },
-            "timeout": 5,
+            "Accept": "application/vnd.github+json",
+            "Authorization": f"token {self.github_user_token.access_token}",
         }
 
     @cached_property
     def installations(self):
-        response = requests.get(
+        response = httpx.get(
             "https://api.github.com/user/installations",
-            **self.github_request_kwargs,
+            headers=self.github_request_headers,
+            timeout=10,
         )
         response.raise_for_status()
         return response.json()["installations"]
@@ -165,7 +167,22 @@ class RepositoriesList(
 ):
     raise_exception = True
 
-    def get_repos(self, *, installation_id):
+    async def _get_page_github_repos(
+        self, *, installation_id, page, per_page, semaphore, httpx_client
+    ):
+        async with semaphore:
+            response = await httpx_client.get(
+                f"https://api.github.com/user/installations/{installation_id}/repositories",
+                params={"per_page": per_page, "page": page},
+                headers=self.github_request_headers,
+            )
+            response.raise_for_status()
+
+            return response.json()
+
+    async def _get_installation_github_repos(
+        self, *, installation_id, semaphore, httpx_client
+    ):
         """
         Get the repositories for this users installation
 
@@ -174,30 +191,67 @@ def get_repos(self, *, installation_id):
         """
         per_page = 100
 
-        def get_page(*, page):
-            return requests.get(
-                f"https://api.github.com/user/installations/{installation_id}/repositories",
-                params={"per_page": per_page, "page": page},
-                **self.github_request_kwargs,
-            ).json()
+        response = await self._get_page_github_repos(
+            installation_id=installation_id,
+            page=1,
+            per_page=per_page,
+            semaphore=semaphore,
+            httpx_client=httpx_client,
+        )
 
-        response = get_page(page=1)
         repos = [repo["full_name"] for repo in response["repositories"]]
 
         remaining_pages = (response["total_count"] - 1) // per_page
 
-        for ii in range(remaining_pages):
-            repos += [
-                repo["full_name"]
-                for repo in get_page(page=ii + 2)["repositories"]
-            ]
+        tasks = []
+
+        async with asyncio.TaskGroup() as task_group:
+            for ii in range(remaining_pages):
+                page = ii + 2
+                tasks.append(
+                    task_group.create_task(
+                        self._get_page_github_repos(
+                            installation_id=installation_id,
+                            page=page,
+                            per_page=per_page,
+                            semaphore=semaphore,
+                            httpx_client=httpx_client,
+                        )
+                    )
+                )
+
+        for task in tasks:
+            response = task.result()
+            repos += [repo["full_name"] for repo in response["repositories"]]
 
         return repos
 
-    def get_list(self):
+    @async_to_sync
+    async def _get_all_github_repos(self):
+        semaphore = asyncio.Semaphore(ASYNC_CONCURRENCY)
+        timeout = httpx.Timeout(10)
+
+        tasks = []
+
+        async with httpx.AsyncClient(timeout=timeout) as httpx_client:
+            async with asyncio.TaskGroup() as task_group:
+                for installation in self.installations:
+                    tasks.append(
+                        task_group.create_task(
+                            self._get_installation_github_repos(
+                                installation_id=installation["id"],
+                                semaphore=semaphore,
+                                httpx_client=httpx_client,
+                            )
+                        )
+                    )
+
         repos = []
 
-        for installation in self.installations:
-            repos += self.get_repos(installation_id=installation["id"])
+        for task in tasks:
+            repos += task.result()
 
         return repos
+
+    def get_list(self):
+        return self._get_all_github_repos()