Merge pull request #2452 from ranaroussi/fix/user-agent

Disable setting user-agent, curl_cffi does that

Author: ValueRaider

yfinance/data.py

@@ -1,5 +1,4 @@
 import functools
-import random
 from functools import lru_cache
 
 from curl_cffi import requests
@@ -11,7 +10,6 @@
 from . import utils, cache
 import threading
 
-from .const import USER_AGENTS
 from .exceptions import YFRateLimitError
 
 cache_maxsize = 64
@@ -66,9 +64,6 @@ class YfData(metaclass=SingletonMeta):
     Have one place to retrieve data from Yahoo API in order to ease caching and speed up operations.
     Singleton means one session one cookie shared by all threads.
     """
-    user_agent_headers = {
-        'User-Agent': random.choice(USER_AGENTS)
-    }
 
     def __init__(self, session=None, proxy=None):
         self._crumb = None
@@ -85,8 +80,6 @@ def __init__(self, session=None, proxy=None):
         self._set_session(session or requests.Session(impersonate="chrome"))
         self._set_proxy(proxy)
 
-        utils.get_yf_logger().debug(f"Using User-Agent: {self.user_agent_headers['User-Agent']}")
-
     def _set_session(self, session):
         if session is None:
             return
@@ -178,7 +171,6 @@ def _get_cookie_basic(self, timeout=30):
         # - 'allow_redirects' copied from @psychoz971 solution - does it help USA?
         self._session.get(
             url='https://fc.yahoo.com',
-            headers=self.user_agent_headers,
             timeout=timeout,
             allow_redirects=True)
 
@@ -191,7 +183,6 @@ def _get_crumb_basic(self, timeout=30):
         # - 'allow_redirects' copied from @psychoz971 solution - does it help USA?
         get_args = {
             'url': "https://query1.finance.yahoo.com/v1/test/getcrumb",
-            'headers': self.user_agent_headers,
             'timeout': timeout,
             'allow_redirects': True
         }
@@ -225,7 +216,6 @@ def _get_cookie_csrf(self, timeout):
             return True
 
         base_args = {
-            'headers': self.user_agent_headers,
             'timeout': timeout}
 
         get_args = {**base_args, 'url': 'https://guce.yahoo.com/consent'}
@@ -297,7 +287,6 @@ def _get_crumb_csrf(self, timeout=30):
 
         get_args = {
             'url': 'https://query2.finance.yahoo.com/v1/test/getcrumb',
-            'headers': self.user_agent_headers,
             'timeout': timeout}
         if self._session_is_caching:
             get_args['expire_after'] = self._expire_after
@@ -337,15 +326,15 @@ def _get_cookie_and_crumb(self, timeout=30):
         return crumb, strategy
 
     @utils.log_indent_decorator
-    def get(self, url, user_agent_headers=None, params=None, timeout=30):
-        return self._make_request(url, request_method = self._session.get, user_agent_headers=user_agent_headers, params=params, timeout=timeout)
+    def get(self, url, params=None, timeout=30):
+        return self._make_request(url, request_method = self._session.get, params=params, timeout=timeout)
 
     @utils.log_indent_decorator
-    def post(self, url, body, user_agent_headers=None, params=None, timeout=30):
-        return self._make_request(url, request_method = self._session.post, user_agent_headers=user_agent_headers, body=body, params=params, timeout=timeout)
+    def post(self, url, body, params=None, timeout=30):
+        return self._make_request(url, request_method = self._session.post, body=body, params=params, timeout=timeout)
 
     @utils.log_indent_decorator
-    def _make_request(self, url, request_method, user_agent_headers=None, body=None, params=None, timeout=30):
+    def _make_request(self, url, request_method, body=None, params=None, timeout=30):
         # Important: treat input arguments as immutable.
 
         if len(url) > 200:
@@ -368,8 +357,7 @@ def _make_request(self, url, request_method, user_agent_headers=None, body=None,
         request_args = {
             'url': url,
             'params': {**params, **crumbs},
-            'timeout': timeout,
-            'headers': user_agent_headers or self.user_agent_headers
+            'timeout': timeout
         }
 
         if body:
@@ -396,11 +384,11 @@ def _make_request(self, url, request_method, user_agent_headers=None, body=None,
 
     @lru_cache_freezeargs
     @lru_cache(maxsize=cache_maxsize)
-    def cache_get(self, url, user_agent_headers=None, params=None, timeout=30):
-        return self.get(url, user_agent_headers, params, timeout)
+    def cache_get(self, url, params=None, timeout=30):
+        return self.get(url, params, timeout)
 
-    def get_raw_json(self, url, user_agent_headers=None, params=None, timeout=30):
+    def get_raw_json(self, url, params=None, timeout=30):
         utils.get_yf_logger().debug(f'get_raw_json(): {url}')
-        response = self.get(url, user_agent_headers=user_agent_headers, params=params, timeout=timeout)
+        response = self.get(url, params=params, timeout=timeout)
         response.raise_for_status()
         return response.json()

yfinance/domain/domain.py

@@ -122,7 +122,7 @@ def _fetch(self, query_url) -> Dict:
             Dict: The JSON response data from the request.
         """
         params_dict = {"formatted": "true", "withReturns": "true", "lang": "en-US", "region": "US"}
-        result = self._data.get_raw_json(query_url, user_agent_headers=self._data.user_agent_headers, params=params_dict)
+        result = self._data.get_raw_json(query_url, params=params_dict)
         return result
 
     def _parse_and_assign_common(self, data) -> None:

yfinance/scrapers/analysis.py

@@ -177,7 +177,7 @@ def _fetch(self, modules: list):
             raise YFException("No valid modules provided, see available modules using `valid_modules`")
         params_dict = {"modules": modules, "corsDomain": "finance.yahoo.com", "formatted": "false", "symbol": self._symbol}
         try:
-            result = self._data.get_raw_json(_QUOTE_SUMMARY_URL_ + f"/{self._symbol}", user_agent_headers=self._data.user_agent_headers, params=params_dict)
+            result = self._data.get_raw_json(_QUOTE_SUMMARY_URL_ + f"/{self._symbol}", params=params_dict)
         except requests.exceptions.HTTPError as e:
             utils.get_yf_logger().error(str(e))
             return None

yfinance/scrapers/funds.py

@@ -175,7 +175,7 @@ def _fetch(self):
         """
         modules = ','.join(["quoteType", "summaryProfile", "topHoldings", "fundProfile"])
         params_dict = {"modules": modules, "corsDomain": "finance.yahoo.com", "symbol": self._symbol, "formatted": "false"}
-        result = self._data.get_raw_json(_QUOTE_SUMMARY_URL_+self._symbol, user_agent_headers=self._data.user_agent_headers, params=params_dict)
+        result = self._data.get_raw_json(_QUOTE_SUMMARY_URL_+self._symbol, params=params_dict)
         return result
 
     def _fetch_and_parse(self) -> None:

yfinance/scrapers/holders.py

@@ -67,7 +67,7 @@ def _fetch(self):
         modules = ','.join(
             ["institutionOwnership", "fundOwnership", "majorDirectHolders", "majorHoldersBreakdown", "insiderTransactions", "insiderHolders", "netSharePurchaseActivity"])
         params_dict = {"modules": modules, "corsDomain": "finance.yahoo.com", "formatted": "false"}
-        result = self._data.get_raw_json(f"{_QUOTE_SUMMARY_URL_}/{self._symbol}", user_agent_headers=self._data.user_agent_headers, params=params_dict)
+        result = self._data.get_raw_json(f"{_QUOTE_SUMMARY_URL_}/{self._symbol}", params=params_dict)
         return result
 
     def _fetch_and_parse(self):

yfinance/scrapers/quote.py

@@ -587,7 +587,7 @@ def _fetch(self, modules: list):
             raise YFException("No valid modules provided, see available modules using `valid_modules`")
         params_dict = {"modules": modules, "corsDomain": "finance.yahoo.com", "formatted": "false", "symbol": self._symbol}
         try:
-            result = self._data.get_raw_json(_QUOTE_SUMMARY_URL_ + f"/{self._symbol}", user_agent_headers=self._data.user_agent_headers, params=params_dict)
+            result = self._data.get_raw_json(_QUOTE_SUMMARY_URL_ + f"/{self._symbol}", params=params_dict)
         except requests.exceptions.HTTPError as e:
             utils.get_yf_logger().error(str(e))
             return None
@@ -596,9 +596,7 @@ def _fetch(self, modules: list):
     def _fetch_additional_info(self):
         params_dict = {"symbols": self._symbol, "formatted": "false"}
         try:
-            result = self._data.get_raw_json(f"{_QUERY1_URL_}/v7/finance/quote?",
-                                             user_agent_headers=self._data.user_agent_headers,
-                                             params=params_dict)
+            result = self._data.get_raw_json(f"{_QUERY1_URL_}/v7/finance/quote?", params=params_dict)
         except requests.exceptions.HTTPError as e:
             utils.get_yf_logger().error(str(e))
             return None
@@ -679,7 +677,7 @@ def _fetch_complementary(self):
             # p = _re.compile(r'root\.App\.main = (.*);')
             # url = 'https://finance.yahoo.com/quote/{}/key-statistics?p={}'.format(self._ticker.ticker, self._ticker.ticker)
             # try:
-            #     r = session.get(url, headers=utils.user_agent_headers)
+            #     r = session.get(url)
             #     data = _json.loads(p.findall(r.text)[0])
             #     key_stats = data['context']['dispatcher']['stores']['QuoteTimeSeriesStore']["timeSeries"]
             #     for k in keys:

yfinance/screener/screener.py

@@ -190,7 +190,6 @@ def screen(query: Union[str, EquityQuery, FundQuery],
     # Fetch
     response = _data.post(_SCREENER_URL_, 
                             body=post_query, 
-                            user_agent_headers=_data.user_agent_headers, 
                             params=params_dict)
     response.raise_for_status()
     return response.json()['finance']['result'][0]

yfinance/utils.py

@@ -40,9 +40,6 @@
 
 from yfinance import const
 
-user_agent_headers = {
-    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'}
-
 # From https://stackoverflow.com/a/59128615
 def attributes(obj):
     disallowed_names = {