Merge pull request #2430 from bretsky/fix/curl-cffi-cookies

Incorporate curl_cffi to avoid rate limiting and cookie errors

Author: ValueRaider

meta.yaml

@@ -26,6 +26,7 @@ requirements:
     - frozendict >=2.3.4
     - beautifulsoup4 >=4.11.1
     - html5lib >=1.1
+    - curl_cffi >=0.7
     - peewee >=3.16.2
     - pip
     - python
@@ -41,6 +42,7 @@ requirements:
     - frozendict >=2.3.4
     - beautifulsoup4 >=4.11.1
     - html5lib >=1.1
+    - curl_cffi >=0.7
     - peewee >=3.16.2
     - python
 

requirements.txt

@@ -9,4 +9,5 @@ beautifulsoup4>=4.11.1
 peewee>=3.16.2
 requests_cache>=1.0
 requests_ratelimiter>=0.3.1
-scipy>=1.6.3
+scipy>=1.6.3
+curl_cffi>=0.7

setup.py

@@ -63,7 +63,7 @@
                       'requests>=2.31', 'multitasking>=0.0.7',
                       'platformdirs>=2.0.0', 'pytz>=2022.5',
                       'frozendict>=2.3.4', 'peewee>=3.16.2',
-                      'beautifulsoup4>=4.11.1'],
+                      'beautifulsoup4>=4.11.1', 'curl_cffi>=0.7'],
     extras_require={
         'nospam': ['requests_cache>=1.0', 'requests_ratelimiter>=0.3.1'],
         'repair': ['scipy>=1.6.3'],

yfinance/base.py

@@ -28,7 +28,8 @@
 
 import numpy as np
 import pandas as pd
-import requests
+from curl_cffi import requests
+
 
 from . import utils, cache
 from .data import YfData
@@ -48,7 +49,7 @@
 class TickerBase:
     def __init__(self, ticker, session=None, proxy=_SENTINEL_):
         self.ticker = ticker.upper()
-        self.session = session
+        self.session = session or requests.Session(impersonate="chrome")
         self._tz = None
 
         self._isin = None

yfinance/data.py

@@ -2,7 +2,7 @@
 import random
 from functools import lru_cache
 
-import requests as requests
+from curl_cffi import requests
 from bs4 import BeautifulSoup
 import datetime
 
@@ -82,7 +82,7 @@ def __init__(self, session=None, proxy=None):
         self._cookie_lock = threading.Lock()
 
         self._session, self._proxy = None, None
-        self._set_session(session or requests.Session())
+        self._set_session(session or requests.Session(impersonate="chrome"))
         self._set_proxy(proxy)
 
         utils.get_yf_logger().debug(f"Using User-Agent: {self.user_agent_headers['User-Agent']}")
@@ -174,14 +174,6 @@ def _load_cookie_basic(self):
         return cookie_dict['cookie']
 
     def _get_cookie_basic(self, timeout=30):
-        if self._cookie is not None:
-            utils.get_yf_logger().debug('reusing cookie')
-            return self._cookie
-
-        self._cookie = self._load_cookie_basic()
-        if self._cookie is not None:
-            return self._cookie
-
         # To avoid infinite recursion, do NOT use self.get()
         # - 'allow_redirects' copied from @psychoz971 solution - does it help USA?
         response = self._session.get(
@@ -190,31 +182,16 @@ def _get_cookie_basic(self, timeout=30):
             timeout=timeout,
             allow_redirects=True)
 
-        if not response.cookies:
-            utils.get_yf_logger().debug("response.cookies = None")
-            return None
-        self._cookie = list(response.cookies)[0]
-        if self._cookie == '':
-            utils.get_yf_logger().debug("list(response.cookies)[0] = ''")
-            return None
-        self._save_cookie_basic(self._cookie)
-        utils.get_yf_logger().debug(f"fetched basic cookie = {self._cookie}")
-        return self._cookie
-
     def _get_crumb_basic(self, timeout=30):
         if self._crumb is not None:
             utils.get_yf_logger().debug('reusing crumb')
             return self._crumb
 
-        cookie = self._get_cookie_basic()
-        if cookie is None:
-            return None
-
+        self._get_cookie_basic()        
         # - 'allow_redirects' copied from @psychoz971 solution - does it help USA?
         get_args = {
             'url': "https://query1.finance.yahoo.com/v1/test/getcrumb",
             'headers': self.user_agent_headers,
-            'cookies': {cookie.name: cookie.value},
             'timeout': timeout,
             'allow_redirects': True
         }
@@ -233,9 +210,9 @@ def _get_crumb_basic(self, timeout=30):
 
     @utils.log_indent_decorator
     def _get_cookie_and_crumb_basic(self, timeout):
-        cookie = self._get_cookie_basic(timeout)
+        self._get_cookie_basic(timeout)
         crumb = self._get_crumb_basic(timeout)
-        return cookie, crumb
+        return crumb
 
     def _get_cookie_csrf(self, timeout):
         if self._cookie is not None:
@@ -338,7 +315,7 @@ def _get_crumb_csrf(self, timeout=30):
 
     @utils.log_indent_decorator
     def _get_cookie_and_crumb(self, timeout=30):
-        cookie, crumb, strategy = None, None, None
+        crumb, strategy = None, None
 
         utils.get_yf_logger().debug(f"cookie_mode = '{self._cookie_strategy}'")
 
@@ -348,16 +325,16 @@ def _get_cookie_and_crumb(self, timeout=30):
                 if crumb is None:
                     # Fail
                     self._set_cookie_strategy('basic', have_lock=True)
-                    cookie, crumb = self._get_cookie_and_crumb_basic(timeout)
+                    crumb = self._get_cookie_and_crumb_basic(timeout)
             else:
                 # Fallback strategy
-                cookie, crumb = self._get_cookie_and_crumb_basic(timeout)
-                if cookie is None or crumb is None:
+                crumb = self._get_cookie_and_crumb_basic(timeout)
+                if crumb is None:
                     # Fail
                     self._set_cookie_strategy('csrf', have_lock=True)
                     crumb = self._get_crumb_csrf()
             strategy = self._cookie_strategy
-        return cookie, crumb, strategy
+        return crumb, strategy
 
     @utils.log_indent_decorator
     def get(self, url, user_agent_headers=None, params=None, timeout=30):
@@ -382,21 +359,15 @@ def _make_request(self, url, request_method, user_agent_headers=None, body=None,
         if 'crumb' in params:
             raise Exception("Don't manually add 'crumb' to params dict, let data.py handle it")
 
-        cookie, crumb, strategy = self._get_cookie_and_crumb()
+        crumb, strategy = self._get_cookie_and_crumb()
         if crumb is not None:
             crumbs = {'crumb': crumb}
         else:
             crumbs = {}
-        if strategy == 'basic' and cookie is not None:
-            # Basic cookie strategy adds cookie to GET parameters
-            cookies = {cookie.name: cookie.value}
-        else:
-            cookies = None
 
         request_args = {
             'url': url,
             'params': {**params, **crumbs},
-            'cookies': cookies,
             'timeout': timeout,
             'headers': user_agent_headers or self.user_agent_headers
         }
@@ -412,10 +383,8 @@ def _make_request(self, url, request_method, user_agent_headers=None, body=None,
                 self._set_cookie_strategy('csrf')
             else:
                 self._set_cookie_strategy('basic')
-            cookie, crumb, strategy = self._get_cookie_and_crumb(timeout)
+            crumb, strategy = self._get_cookie_and_crumb(timeout)
             request_args['params']['crumb'] = crumb
-            if strategy == 'basic':
-                request_args['cookies'] = {cookie.name: cookie.value}
             response = request_method(**request_args)
             utils.get_yf_logger().debug(f'response code={response.status_code}')
 

yfinance/multi.py

@@ -28,6 +28,7 @@
 
 import multitasking as _multitasking
 import pandas as _pd
+from curl_cffi import requests
 
 from . import Ticker, utils
 from .data import YfData
@@ -90,6 +91,7 @@ def download(tickers, start=None, end=None, actions=False, threads=True,
             Optional. Always return a MultiIndex DataFrame? Default is True
     """
     logger = utils.get_yf_logger()
+    session = session or requests.Session(impersonate="chrome")
 
     if auto_adjust is None:
         # Warn users that default has changed to True

yfinance/scrapers/history.py

@@ -6,6 +6,7 @@
 from math import isclose
 import time as _time
 import bisect
+from curl_cffi import requests
 
 from yfinance import shared, utils
 from yfinance.const import _BASE_URL_, _PRICE_COLNAMES_, _SENTINEL_
@@ -19,7 +20,7 @@ def __init__(self, data, ticker, tz, session=None, proxy=_SENTINEL_):
         if proxy is not _SENTINEL_:
             utils.print_once("YF deprecation warning: set proxy via new config function: yf.set_config(proxy=proxy)")
             self._data._set_proxy(proxy)
-        self.session = session
+        self.session = session or requests.Session(impersonate="chrome")
 
         self._history_cache = {}
         self._history_metadata = None

yfinance/utils.py

@@ -35,7 +35,6 @@
 import numpy as _np
 import pandas as _pd
 import pytz as _tz
-import requests as _requests
 from dateutil.relativedelta import relativedelta
 from pytz import UnknownTimeZoneError
 
@@ -196,7 +195,6 @@ def get_all_by_isin(isin, proxy=const._SENTINEL_, session=None):
     # Deferred this to prevent circular imports
     from .search import Search
 
-    session = session or _requests.Session()
     search = Search(query=isin, max_results=1, session=session, proxy=proxy)
 
     # Extract the first quote and news