55retry logic and proper timeout handling.
66"""
77
8+ from typing import Any , Mapping , Optional , Union
9+
810import requests
911from requests .adapters import HTTPAdapter
12+ from requests .models import PreparedRequest , Response
1013from urllib3 .util .retry import Retry
1114
1215
16+ class TimeoutHTTPAdapter (HTTPAdapter ):
17+ def __init__ (self , * args : Any , ** kwargs : Any ) -> None :
18+ self .timeout = kwargs .pop ("timeout" , None )
19+ super ().__init__ (* args , ** kwargs )
20+
21+ def send (
22+ self ,
23+ request : PreparedRequest ,
24+ stream : bool = False ,
25+ timeout : Union [float , tuple [float , float ], tuple [float , None ], None ] = None ,
26+ verify : Union [bool , str ] = True ,
27+ cert : Union [bytes , str , tuple [Union [bytes , str ], Union [bytes , str ]], None ] = None ,
28+ proxies : Optional [Mapping [str , str ]] = None ,
29+ ) -> Response :
30+ timeout = timeout or self .timeout
31+ return super ().send (
32+ request , stream = stream , timeout = timeout , verify = verify , cert = cert , proxies = proxies
33+ )
34+
35+
1336def create_http_session () -> requests .Session :
1437 """Create a new HTTP session with retry logic for rate limits and server errors."""
1538 session = requests .Session ()
@@ -21,12 +44,8 @@ def create_http_session() -> requests.Session:
2144 status_forcelist = [429 , 500 , 502 , 503 , 504 ],
2245 )
2346
24- adapter = HTTPAdapter ( max_retries = retry_strategy )
47+ adapter = TimeoutHTTPAdapter ( timeout = ( 10 , 30 ), max_retries = retry_strategy )
2548 session .mount ("http://" , adapter )
2649 session .mount ("https://" , adapter )
2750
28- # Set standard timeouts (connect_timeout, read_timeout)
29- # Note: Session.timeout is not a standard attribute, but we'll add it as a custom attribute
30- session .timeout = (10 , 60 ) # type: ignore[attr-defined] # 10s connection, 60s read for downloads
31-
3251 return session
0 commit comments