From 58badc443c4f7c06b73f633bac81c90f965b494d Mon Sep 17 00:00:00 2001 From: Peter Law Date: Sun, 23 Jan 2022 15:57:42 +0000 Subject: [PATCH 1/5] Add support for retrying when GitHub rate limit the import --- scripts/import_backends.py | 40 +++++++++++++++++++++++++++++++----- scripts/requirements-dev.txt | 7 ++++++- scripts/requirements.in | 1 + scripts/requirements.txt | 4 ++++ 4 files changed, 46 insertions(+), 6 deletions(-) diff --git a/scripts/import_backends.py b/scripts/import_backends.py index 56ff4cd..c97446a 100644 --- a/scripts/import_backends.py +++ b/scripts/import_backends.py @@ -6,6 +6,7 @@ from typing import TYPE_CHECKING, Dict, List, Sequence, cast import github +import retrying from termcolor import cprint from ticket_type import Ticket @@ -224,12 +225,41 @@ def submit(self, ticket: Ticket) -> int: # larger gap for better reliability. time.sleep(5) - issue = self.repo.create_issue( - ticket.summary, - self.description_text(ticket), - milestone=self.get_or_create_milestone(ticket.milestone), - labels=self.labels(ticket), + # We also want to wait and retry when GitHub then additionally rate + # limit us anyway... + + def retry_on_exception(exception: Exception) -> bool: + return isinstance(exception, github.GithubException) + + @retrying.retry( + retry_on_exception=retry_on_exception, + wait_fixed=20_000, ) + def create_issue() -> github.Issue.Issue: + RATE_LIMIT_MESSAGE = "exceeded a secondary rate limit and have been temporarily blocked" # noqa:E501 + + try: + return self.repo.create_issue( + ticket.summary, + self.description_text(ticket), + milestone=self.get_or_create_milestone(ticket.milestone), + labels=self.labels(ticket), + ) + except github.GithubException as e: + message = e.data.get('message') + if ( + isinstance(message, str) + and RATE_LIMIT_MESSAGE in message + ): + print("... GitHub Rate Limited ...") + raise github.RateLimitExceededException( + status=e.status, + data=e.data, + headers=e.headers, + ) from e + raise + + issue = create_issue() ticket_number: int = issue.number self._known_titles[ticket_number] = ticket.summary diff --git a/scripts/requirements-dev.txt b/scripts/requirements-dev.txt index cd6bbca..3afaf3f 100644 --- a/scripts/requirements-dev.txt +++ b/scripts/requirements-dev.txt @@ -89,8 +89,13 @@ requests==2.27.1 # via # -r ./scripts/requirements.txt # pygithub +retrying==1.3.3 + # via -r ./scripts/requirements.txt six==1.16.0 - # via flake8-tuple + # via + # -r ./scripts/requirements.txt + # flake8-tuple + # retrying termcolor==1.1.0 # via -r ./scripts/requirements.txt testfixtures==6.18.5 diff --git a/scripts/requirements.in b/scripts/requirements.in index b340c3e..8e1523e 100644 --- a/scripts/requirements.in +++ b/scripts/requirements.in @@ -1,3 +1,4 @@ PyGithub PyYAML +retrying termcolor diff --git a/scripts/requirements.txt b/scripts/requirements.txt index 18f8674..a6c322e 100644 --- a/scripts/requirements.txt +++ b/scripts/requirements.txt @@ -26,6 +26,10 @@ pyyaml==6.0 # via -r ./scripts/requirements.in requests==2.27.1 # via pygithub +retrying==1.3.3 + # via -r ./scripts/requirements.in +six==1.16.0 + # via retrying termcolor==1.1.0 # via -r ./scripts/requirements.in urllib3==1.26.9 From 46058264858c2a2dc8d31f3c55890b9f6e7791d6 Mon Sep 17 00:00:00 2001 From: Peter Law Date: Sun, 23 Jan 2022 16:38:19 +0000 Subject: [PATCH 2/5] Ingore lack of retrying type annotations --- scripts/import_backends.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/import_backends.py b/scripts/import_backends.py index c97446a..311a814 100644 --- a/scripts/import_backends.py +++ b/scripts/import_backends.py @@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, Dict, List, Sequence, cast import github -import retrying +import retrying # type: ignore[import] from termcolor import cprint from ticket_type import Ticket From e7528daeb8f0786c8258b9a18c2ea2e9339b4c9a Mon Sep 17 00:00:00 2001 From: Peter Law Date: Sun, 23 Jan 2022 16:43:12 +0000 Subject: [PATCH 3/5] Make the delays longer to see if this helps --- scripts/import_backends.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/import_backends.py b/scripts/import_backends.py index 311a814..60f92f2 100644 --- a/scripts/import_backends.py +++ b/scripts/import_backends.py @@ -223,7 +223,7 @@ def submit(self, ticket: Ticket) -> int: # primary rate limits which can be queried via the API. Manual testing # indicates that ~2s isn't always sufficient, so err on the side of a # larger gap for better reliability. - time.sleep(5) + time.sleep(10) # We also want to wait and retry when GitHub then additionally rate # limit us anyway... @@ -233,7 +233,7 @@ def retry_on_exception(exception: Exception) -> bool: @retrying.retry( retry_on_exception=retry_on_exception, - wait_fixed=20_000, + wait_fixed=60_000, ) def create_issue() -> github.Issue.Issue: RATE_LIMIT_MESSAGE = "exceeded a secondary rate limit and have been temporarily blocked" # noqa:E501 From 91266ed73a107f28baeb6cc410e5ae0f166fdc1a Mon Sep 17 00:00:00 2001 From: Peter Law Date: Thu, 27 Jan 2022 00:16:37 +0000 Subject: [PATCH 4/5] Only retry for rate limit errors Other kinds of errors won't go away after just waiting for a while. --- scripts/import_backends.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/import_backends.py b/scripts/import_backends.py index 60f92f2..a311741 100644 --- a/scripts/import_backends.py +++ b/scripts/import_backends.py @@ -229,7 +229,7 @@ def submit(self, ticket: Ticket) -> int: # limit us anyway... def retry_on_exception(exception: Exception) -> bool: - return isinstance(exception, github.GithubException) + return isinstance(exception, github.RateLimitExceededException) @retrying.retry( retry_on_exception=retry_on_exception, From f7aa6594a3f383ac9986170dd4d6e131f04a11b5 Mon Sep 17 00:00:00 2001 From: Peter Law Date: Thu, 27 Jan 2022 00:17:15 +0000 Subject: [PATCH 5/5] Print for all rate limit errors Not just the ones that we're intercepting and converting. --- scripts/import_backends.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/import_backends.py b/scripts/import_backends.py index a311741..816fe75 100644 --- a/scripts/import_backends.py +++ b/scripts/import_backends.py @@ -229,7 +229,10 @@ def submit(self, ticket: Ticket) -> int: # limit us anyway... def retry_on_exception(exception: Exception) -> bool: - return isinstance(exception, github.RateLimitExceededException) + if isinstance(exception, github.RateLimitExceededException): + print("... GitHub Rate Limited ...") + return True + return False @retrying.retry( retry_on_exception=retry_on_exception, @@ -251,7 +254,6 @@ def create_issue() -> github.Issue.Issue: isinstance(message, str) and RATE_LIMIT_MESSAGE in message ): - print("... GitHub Rate Limited ...") raise github.RateLimitExceededException( status=e.status, data=e.data,