[twitter] implement 'retries-api' option (#8317)

retry API requests when encountering server-related errors

Author: mikf

docs/configuration.rst

@@ -6294,6 +6294,18 @@ Note
     <extractor.*.image-filter_>`__.
 
 
+extractor.twitter.retries-api
+-----------------------------
+Type
+    ``integer``
+Default
+    ``4``
+Description
+    Maximum number of retries
+    for API requests when encountering server ``errors``,
+    or ``-1`` for infinite retries.
+
+
 extractor.twitter.retweets
 --------------------------
 Type

docs/gallery-dl.conf

@@ -877,6 +877,7 @@
             "quoted"      : false,
             "ratelimit"   : "wait",
             "replies"     : true,
+            "retries-api" : 4,
             "retweets"    : false,
             "search-limit": 20,
             "search-pagination": "cursor",

gallery_dl/extractor/twitter.py

@@ -1902,14 +1902,14 @@ def _pagination_tweets(self, endpoint, variables,
 
         while True:
             params["variables"] = self._json_dumps(variables)
-            data = self._call(endpoint, params)["data"]
+            data = self._call(endpoint, params)
 
             try:
                 if path is None:
-                    instructions = (data["user"]["result"]["timeline"]
+                    instructions = (data["data"]["user"]["result"]["timeline"]
                                     ["timeline"]["instructions"])
                 else:
-                    instructions = data
+                    instructions = data["data"]
                     for key in path:
                         instructions = instructions[key]
                     instructions = instructions["instructions"]
@@ -1940,6 +1940,26 @@ def _pagination_tweets(self, endpoint, variables,
             except LookupError:
                 extr.log.debug(data)
 
+                if errors := data.get("errors"):
+                    if "api_retries" not in locals():
+                        api_tries = 1
+                        api_retries = extr.config("retries-api", 4)
+                        if api_retries < 0:
+                            api_retries = float("inf")
+
+                    err = []
+                    srv = False
+                    for e in errors:
+                        err.append(f"- '{e.get('message') or e.get('name')}'")
+                        if e.get("source") == "Server":
+                            srv = True
+
+                    self.log.warning("API errors (%s/%s):\n%s",
+                                     api_tries, api_retries+1, "\n".join(err))
+                    if srv and api_tries <= api_retries:
+                        api_tries += 1
+                        continue
+
                 if user := extr._user_obj:
                     user = user["legacy"]
                     if user.get("blocked_by"):