diff --git a/scripts/import_backends.py b/scripts/import_backends.py index 56ff4cd..816fe75 100644 --- a/scripts/import_backends.py +++ b/scripts/import_backends.py @@ -6,6 +6,7 @@ from typing import TYPE_CHECKING, Dict, List, Sequence, cast import github +import retrying # type: ignore[import] from termcolor import cprint from ticket_type import Ticket @@ -222,14 +223,45 @@ def submit(self, ticket: Ticket) -> int: # primary rate limits which can be queried via the API. Manual testing # indicates that ~2s isn't always sufficient, so err on the side of a # larger gap for better reliability. - time.sleep(5) + time.sleep(10) - issue = self.repo.create_issue( - ticket.summary, - self.description_text(ticket), - milestone=self.get_or_create_milestone(ticket.milestone), - labels=self.labels(ticket), + # We also want to wait and retry when GitHub then additionally rate + # limit us anyway... + + def retry_on_exception(exception: Exception) -> bool: + if isinstance(exception, github.RateLimitExceededException): + print("... GitHub Rate Limited ...") + return True + return False + + @retrying.retry( + retry_on_exception=retry_on_exception, + wait_fixed=60_000, ) + def create_issue() -> github.Issue.Issue: + RATE_LIMIT_MESSAGE = "exceeded a secondary rate limit and have been temporarily blocked" # noqa:E501 + + try: + return self.repo.create_issue( + ticket.summary, + self.description_text(ticket), + milestone=self.get_or_create_milestone(ticket.milestone), + labels=self.labels(ticket), + ) + except github.GithubException as e: + message = e.data.get('message') + if ( + isinstance(message, str) + and RATE_LIMIT_MESSAGE in message + ): + raise github.RateLimitExceededException( + status=e.status, + data=e.data, + headers=e.headers, + ) from e + raise + + issue = create_issue() ticket_number: int = issue.number self._known_titles[ticket_number] = ticket.summary diff --git a/scripts/requirements-dev.txt b/scripts/requirements-dev.txt index cd6bbca..3afaf3f 100644 --- a/scripts/requirements-dev.txt +++ b/scripts/requirements-dev.txt @@ -89,8 +89,13 @@ requests==2.27.1 # via # -r ./scripts/requirements.txt # pygithub +retrying==1.3.3 + # via -r ./scripts/requirements.txt six==1.16.0 - # via flake8-tuple + # via + # -r ./scripts/requirements.txt + # flake8-tuple + # retrying termcolor==1.1.0 # via -r ./scripts/requirements.txt testfixtures==6.18.5 diff --git a/scripts/requirements.in b/scripts/requirements.in index b340c3e..8e1523e 100644 --- a/scripts/requirements.in +++ b/scripts/requirements.in @@ -1,3 +1,4 @@ PyGithub PyYAML +retrying termcolor diff --git a/scripts/requirements.txt b/scripts/requirements.txt index 18f8674..a6c322e 100644 --- a/scripts/requirements.txt +++ b/scripts/requirements.txt @@ -26,6 +26,10 @@ pyyaml==6.0 # via -r ./scripts/requirements.in requests==2.27.1 # via pygithub +retrying==1.3.3 + # via -r ./scripts/requirements.in +six==1.16.0 + # via retrying termcolor==1.1.0 # via -r ./scripts/requirements.in urllib3==1.26.9