[Sofascore] Bypass TLS fingerprinting (#835)

Author: probberechts

poetry.lock

@@ -22,7 +22,6 @@ Changelog = "https://github.com/probberechts/soccerdata/releases"
 python = ">=3.9,<3.13"
 PySocks = "^1.7.1"
 Unidecode = "^1.2.0"
-cloudscraper = "^1.2.71"
 html5lib = "^1.1"
 pandas = "^2.0.0, !=2.1.0"
 requests = "^2.23"
@@ -32,6 +31,8 @@ unicode = "^2.7"
 lxml = "^4.9.3"
 socceraction = {version="^1.5.3", optional=true}
 packaging = "^24.1"
+wrapper-tls-requests = "^1.1.4"
+tqdm = "^4.67.1"
 
 [tool.poetry.extras]
 socceraction = ["socceraction"]

pyproject.toml

@@ -12,11 +12,11 @@
 from pathlib import Path
 from typing import IO, Callable, Optional, Union
 
-import cloudscraper
 import numpy as np
 import pandas as pd
 import requests
 import selenium
+import tls_requests
 import undetected_chromedriver as uc
 from dateutil.relativedelta import relativedelta
 from packaging import version
@@ -504,12 +504,14 @@ def __init__(
 
         self._session = self._init_session()
 
-    def _init_session(self) -> requests.Session:
-        session = cloudscraper.create_scraper(
-            browser={"browser": "chrome", "platform": "linux", "mobile": False}
-        )
-        session.proxies.update(self.proxy())
-        return session
+    def _init_session(self) -> tls_requests.Client:
+        proxy = self.proxy()
+        proxy_url = None
+        for protocol in ["https", "http"]:
+            if protocol in proxy:
+                proxy_url = proxy[protocol]
+                break
+        return tls_requests.Client(proxy=proxy_url)
 
     def _download_and_save(
         self,
@@ -520,7 +522,7 @@ def _download_and_save(
         """Download file at url to filepath. Overwrites if filepath exists."""
         for i in range(5):
             try:
-                response = self._session.get(url, stream=True)
+                response = self._session.get(url)
                 time.sleep(self.rate_limit + random.random() * self.max_delay)
                 response.raise_for_status()
                 if var is not None: