Support curl_cffi and fix cookie handling

requirements.txt

@@ -9,4 +9,5 @@ beautifulsoup4>=4.11.1
 peewee>=3.16.2
 requests_cache>=1.0
 requests_ratelimiter>=0.3.1
-scipy>=1.6.3
+scipy>=1.6.3
+curl_cffi>=0.10.0

yfinance/data.py

@@ -3,6 +3,8 @@
 from functools import lru_cache
 
 import requests as requests
+from curl_cffi import requests as curl_cffi_requests
+
 from bs4 import BeautifulSoup
 import datetime
 
@@ -82,7 +84,7 @@ def __init__(self, session=None, proxy=None):
         self._cookie_lock = threading.Lock()
 
         self._session, self._proxy = None, None
-        self._set_session(session or requests.Session())
+        self._set_session(session or curl_cffi_requests.Session(impersonate="chrome"))
         self._set_proxy(proxy)
 
         utils.get_yf_logger().debug(f"Using User-Agent: {self.user_agent_headers['User-Agent']}")
@@ -193,7 +195,11 @@ def _get_cookie_basic(self, timeout=30):
         if not response.cookies:
             utils.get_yf_logger().debug("response.cookies = None")
             return None
-        self._cookie = list(response.cookies)[0]
+        if isinstance(response.cookies, curl_cffi_requests.cookies.Cookies):
+            _cookiejar = utils.convert_curl_cffi_cookies_to_cookiejar(response.cookies)
+        else:
+            _cookiejar = response.cookies
+        self._cookie = list(_cookiejar)[0]
         if self._cookie == '':
             utils.get_yf_logger().debug("list(response.cookies)[0] = ''")
             return None

yfinance/domain/domain.py

@@ -20,7 +20,7 @@ def __init__(self, key: str, session=None, proxy=_SENTINEL_):
 
         Args:
             key (str): Unique key identifying the domain entity.
-            session (Optional[requests.Session]): Session object for HTTP requests. Defaults to None.
+            session (Optional[curl_cffi.requests.Session]): Session object for HTTP requests. Defaults to None.
         """
         self._key: str = key
         self.session = session

yfinance/domain/sector.py

@@ -19,7 +19,7 @@ def __init__(self, key, session=None, proxy=_SENTINEL_):
         """
         Args:
             key (str): The key representing the sector.
-            session (requests.Session, optional): A session for making requests. Defaults to None.
+            session (curl_cffi.requests.Session, optional): A session for making requests. Defaults to None.
             proxy (dict, optional): A dictionary containing proxy settings for the request. Defaults to None.
 
         .. seealso::

yfinance/utils.py

@@ -39,6 +39,8 @@
 from dateutil.relativedelta import relativedelta
 from pytz import UnknownTimeZoneError
 
+from curl_cffi import requests as curl_cffi_requests
+
 from yfinance import const
 
 user_agent_headers = {
@@ -196,7 +198,7 @@ def get_all_by_isin(isin, proxy=const._SENTINEL_, session=None):
     # Deferred this to prevent circular imports
     from .search import Search
 
-    session = session or _requests.Session()
+    session = session or curl_cffi_requests.Session(impersonate="chrome")
     search = Search(query=isin, max_results=1, session=session, proxy=proxy)
 
     # Extract the first quote and news
@@ -1125,3 +1127,22 @@ def generate_list_table_from_dict_universal(data: dict, bullets: bool=True, titl
                 table += ' '*5 + f"- {value_str}\n"
 
     return table
+
+
+def convert_curl_cffi_cookies_to_cookiejar(curl_cffi_cookies, domain=".yahoo.com"):
+    """
+    Convert curl_cffi Cookies object to requests RequestsCookieJar
+    Args:
+        curl_cffi_cookies: curl_cffi.requests.cookies.Cookies instance
+        domain: cookie default domain
+    Returns:
+        RequestsCookieJar instance
+    """
+    from requests.cookies import RequestsCookieJar, create_cookie
+
+    jar = RequestsCookieJar()
+    cookie_items = curl_cffi_cookies.items()
+    for name, value in cookie_items:
+        cookie = create_cookie(name=name, value=value, domain=domain)
+        jar.set_cookie(cookie)
+    return jar