Merge pull request #391 from jdepoix/feature/avoid-open-tcp-connections

Feature/avoid open tcp connections

Author: jdepoix

README.md

@@ -283,7 +283,7 @@ therefore integrated it into this module, to make setting it up as easy as possi
 ### Using [Webshare](https://www.webshare.io/?referral_code=w0xno53eb50g)
 
 Once you have created a [Webshare account](https://www.webshare.io/?referral_code=w0xno53eb50g) and purchased a 
-"Residential Proxy" package that suites your workload, open the 
+"Residential Proxy" package that suits your workload, open the 
 [Webshare Proxy Settings](https://dashboard.webshare.io/proxy/settings) to retrieve your "Proxy Username" and 
 "Proxy Password". Using this information you can initialize the `YouTubeTranscriptApi` as follows:
 
@@ -508,7 +508,7 @@ using residential proxies as explained in
 [Working around IP bans](#working-around-ip-bans-requestblocked-or-ipblocked-exception). To use
 [Webshare residential proxies](https://www.webshare.io/?referral_code=w0xno53eb50g) through the CLI, you will have to 
 create a [Webshare account](https://www.webshare.io/?referral_code=w0xno53eb50g) and purchase a residential 
-proxy package that suites your workload. Then you can use the "Proxy Username" and "Proxy Password" which you can find 
+proxy package that suits your workload. Then you can use the "Proxy Username" and "Proxy Password" which you can find 
 in your [Webshare Proxy Settings](https://dashboard.webshare.io/proxy/settings), to run the following command:
 
 ```

youtube_transcript_api/_api.py

@@ -48,6 +48,8 @@ def __init__(
             http_client.cookies = _load_cookie_jar(cookie_path)
         if proxy_config is not None:
             http_client.proxies = proxy_config.to_requests_dict()
+            if proxy_config.prevent_keeping_connections_alive():
+                http_client.headers.update({"Connection": "close"})
         self._fetcher = TranscriptListFetcher(http_client)
 
     def fetch(
@@ -59,7 +61,7 @@ def fetch(
         """
         Retrieves the transcript for a single video. This is just a shortcut for
         calling:
-        `YouTubeTranscriptApi.list_transcripts(video_id, proxies).find_transcript(languages).fetch()`
+        `YouTubeTranscriptApi().list(video_id).find_transcript(languages).fetch(preserve_formatting=preserve_formatting)`
 
         :param video_id: the ID of the video you want to retrieve the transcript for.
             Make sure that this is the actual ID, NOT the full URL to the video!

youtube_transcript_api/proxies.py

@@ -32,6 +32,14 @@ def to_requests_dict(self) -> RequestsProxyConfigDict:
         """
         pass
 
+    def prevent_keeping_connections_alive(self) -> bool:
+        """
+        If you are using rotating proxies, it can be useful to prevent the HTTP
+        client from keeping TCP connections alive, as your IP won't be rotated on
+        every request, if your connection stays open.
+        """
+        return False
+
 
 class GenericProxyConfig(ProxyConfig):
     """
@@ -76,7 +84,7 @@ class WebshareProxyConfig(GenericProxyConfig):
 
     If you don't have a Webshare account yet, you will have to create one
     at https://www.webshare.io/?referral_code=w0xno53eb50g and purchase a residential
-    proxy package that suites your workload, to be able to use this proxy config.
+    proxy package that suits your workload, to be able to use this proxy config.
 
     Once you have created an account you only need the "Proxy Username" and
     "Proxy Password" that you can find in your Webshare settings
@@ -130,3 +138,6 @@ def http_url(self) -> str:
     @property
     def https_url(self) -> str:
         return self.url
+
+    def prevent_keeping_connections_alive(self) -> bool:
+        return True

youtube_transcript_api/test/test_api.py

@@ -26,7 +26,7 @@
     RequestBlocked,
     VideoUnplayable,
 )
-from youtube_transcript_api.proxies import GenericProxyConfig
+from youtube_transcript_api.proxies import GenericProxyConfig, WebshareProxyConfig
 
 
 def get_asset_path(filename: str) -> Path:
@@ -330,6 +330,17 @@ def test_fetch__with_proxy(self, to_requests_dict):
         )
         to_requests_dict.assert_any_call()
 
+    @patch("youtube_transcript_api.proxies.GenericProxyConfig.to_requests_dict")
+    def test_fetch__with_proxy_prevent_alive_connections(self, to_requests_dict):
+        proxy_config = WebshareProxyConfig(
+            proxy_username="username", proxy_password="password"
+        )
+
+        YouTubeTranscriptApi(proxy_config=proxy_config).fetch("GJLlxj_dtq8")
+
+        request = httpretty.last_request()
+        self.assertEqual(request.headers.get("Connection"), "close")
+
     def test_fetch__with_cookies(self):
         cookie_path = get_asset_path("example_cookies.txt")
         transcript = YouTubeTranscriptApi(cookie_path=cookie_path).fetch("GJLlxj_dtq8")