Release 2.2.1

Author: theypsilon

src/downloader/http_gateway.py

@@ -37,8 +37,10 @@ def debug(self, *args: Any) -> None: ...
 
 
 class HttpConfig(TypedDict):
-    http_proxy: Optional[tuple[str, str, int]]  # (scheme, host, port)
-    https_proxy: Optional[tuple[str, str, int]]  # (scheme, host, port)
+    http_proxy: Optional[ParseResult]
+    https_proxy: Optional[ParseResult]
+    http_proxy_headers: Optional[Dict[str, str]]
+    https_proxy_headers: Optional[Dict[str, str]]
 
 
 class HttpGateway:
@@ -79,13 +81,14 @@ def open(self, url: str, method: Optional[str] = None, body: Any = None, headers
         if self._logger is not None: self._logger.debug(f'^^^^ {method} {url}')
         url = self._process_url(url)
         parsed_url = urlparse(url)
-        if parsed_url.scheme not in {'http', 'https'}: raise HttpGatewayException(f"URL '{url}' has wrong scheme '{parsed_url.scheme}'.")
+        scheme_code = _scheme_dict.get(parsed_url.scheme, -1)
+        if scheme_code == -1: raise HttpGatewayException(f"URL '{url}' has wrong scheme '{parsed_url.scheme}'.")
         final_url, conn = self._request(
             url,
             parsed_url,
             method,
             body,
-            {**_default_headers, **headers} if isinstance(headers, dict) else _default_headers,
+            self._make_headers(headers, is_http=scheme_code==0),
         )
         if self._logger is not None: self._logger.debug(f'HTTP {conn.response.status}: {final_url}\n'
                                                         f'1st byte @ {time.monotonic() - now:.3f}s\nvvvv\n')
@@ -95,6 +98,13 @@ def open(self, url: str, method: Optional[str] = None, body: Any = None, headers
             conn.finish_response()
             if self._logger is not None: self._logger.print(f'|||| Done: {final_url} ({time.monotonic() - now:.3f}s)')
 
+    def _make_headers(self, headers: Any, is_http: bool) -> dict[str, str]:
+        if is_http and self._config and self._config['http_proxy_headers']:
+            headers = headers if isinstance(headers, dict) else {}
+            return {**_default_headers, **headers, **self._config['http_proxy_headers']}
+        else:
+            return {**_default_headers, **headers} if isinstance(headers, dict) else _default_headers
+
     def cleanup(self) -> None:
         self._out_of_service = True
         total_cleared = 0
@@ -245,23 +255,51 @@ def _fill_redirects_swap(self, now: float, lock: threading.Lock, redirects: Dict
         return size != len(self._redirects_swap)
 
 
+_scheme_dict = {
+    'http': 0,
+    'https': 1
+}
+
 def http_config(http_proxy: Optional[str], https_proxy: Optional[str]) -> HttpConfig:
-    config: HttpConfig = {"http_proxy": None, "https_proxy": None}
+    config: HttpConfig = {
+        "http_proxy": None,
+        "https_proxy": None,
+        "http_proxy_headers": None,
+        "https_proxy_headers": None
+    }
+
+    if not http_proxy and not https_proxy: return config
+
     if http_proxy:
         parsed = urlparse(http_proxy)
         if parsed.hostname and parsed.scheme in ('http', 'https'):
-            config['http_proxy'] = (parsed.scheme, parsed.hostname, parsed.port or (443 if parsed.scheme == 'https' else 80))
+            config['http_proxy'] = parsed
+            auth_header = _make_proxy_auth_header(parsed)
+            if auth_header:
+                config['http_proxy_headers'] = {'Proxy-Authorization': auth_header}
 
     if not https_proxy and http_proxy:
         https_proxy = http_proxy
 
     if https_proxy:
         parsed = urlparse(https_proxy)
         if parsed.hostname and parsed.scheme in ('http', 'https'):
-            config['https_proxy'] = (parsed.scheme, parsed.hostname, parsed.port or (443 if parsed.scheme == 'https' else 80))
+            config['https_proxy'] = parsed
+            auth_header = _make_proxy_auth_header(parsed)
+            if auth_header:
+                config['https_proxy_headers'] = {'Proxy-Authorization': auth_header}
 
     return config
 
+
+def _make_proxy_auth_header(proxy: ParseResult) -> Optional[str]:
+    if proxy.username and proxy.password:
+        import base64
+        credentials = f"{proxy.username}:{proxy.password}"
+        encoded = base64.b64encode(credentials.encode()).decode('ascii')
+        return f"Basic {encoded}"
+    return None
+
 USER_AGENT = 'Downloader/2.X (Linux; [email protected])'
 _default_headers = {'User-Agent': USER_AGENT, 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=120'}
 
@@ -420,23 +458,27 @@ def clear_timed_outs(self, now: float) -> int:
 def create_http_connection(scheme: str, netloc: str, timeout: float, ctx: ssl.SSLContext, config: Optional[dict[str, Any]]) -> HTTPConnection:
     if scheme == 'http':
         if config and config['http_proxy']:
-            proxy_scheme, proxy_host, proxy_port = config['http_proxy']
-            if proxy_scheme == 'https':
+            proxy = config['http_proxy']
+            proxy_host = proxy.hostname
+            proxy_port = proxy.port or (443 if proxy.scheme == 'https' else 80)
+            if proxy.scheme == 'https':
                 return HTTPSConnection(proxy_host, proxy_port, timeout=timeout, context=ctx)
             return HTTPConnection(proxy_host, proxy_port, timeout=timeout)
         return HTTPConnection(netloc, timeout=timeout)
 
     elif scheme == 'https':
         if config and config['https_proxy']:
-            proxy_scheme, proxy_host, proxy_port = config['https_proxy']
+            proxy = config['https_proxy']
+            proxy_host = proxy.hostname
+            proxy_port = proxy.port or (443 if proxy.scheme == 'https' else 80)
             parsed_netloc = urlparse(f'//{netloc}')
             target_host = parsed_netloc.hostname
             target_port = parsed_netloc.port or 443
             if not target_host:
                 raise HttpGatewayException(f"Invalid netloc: {netloc}")
 
             conn = HTTPSConnection(proxy_host, proxy_port, timeout=timeout, context=ctx)
-            conn.set_tunnel(target_host, target_port)
+            conn.set_tunnel(target_host, target_port, headers=config.get('https_proxy_headers'))
             return conn
         return HTTPSConnection(netloc, timeout=timeout, context=ctx)
 

src/test/exploratory/http_gateway_connections/explore_http_gateway_with_real_urls.py

@@ -71,23 +71,20 @@ def main() -> None:
 
     http_proxy_url = os.environ.get('HTTP_PROXY')
     https_proxy_url = os.environ.get('HTTPS_PROXY')
-    config = http_config(http_proxy=http_proxy_url, https_proxy=https_proxy_url) if (http_proxy_url or https_proxy_url) else None
-
-    if config:
-        logger.print('PROXY CONFIGURATION:')
-        if config.get('http_proxy'):
-            scheme, host, port = config['http_proxy']
-            logger.print(f'  HTTP Proxy:  {scheme}://{host}:{port}')
-        else:
-            logger.print(f'  HTTP Proxy:  None (direct connection)')
-
-        if config.get('https_proxy'):
-            scheme, host, port = config['https_proxy']
-            logger.print(f'  HTTPS Proxy: {scheme}://{host}:{port}')
-        else:
-            logger.print(f'  HTTPS Proxy: None (direct connection)')
+    config = http_config(http_proxy=http_proxy_url, https_proxy=https_proxy_url)
+
+    logger.print('PROXY CONFIGURATION:')
+    if config.get('http_proxy'):
+        scheme, host, port = config['http_proxy'].scheme, config['http_proxy'].hostname, config['http_proxy'].port
+        logger.print(f'  HTTP Proxy:  {scheme}://{host}:{port}')
+    else:
+        logger.print(f'  HTTP Proxy:  None (direct connection)')
+
+    if config.get('https_proxy'):
+        scheme, host, port = config['http_proxy'].scheme, config['http_proxy'].hostname, config['http_proxy'].port
+        logger.print(f'  HTTPS Proxy: {scheme}://{host}:{port}')
     else:
-        logger.print('NO PROXY - Using direct connection')
+        logger.print(f'  HTTPS Proxy: None (direct connection)')
 
     with HttpGateway(ssl_ctx=ssl.create_default_context(), timeout=180, logger=logger, config=config) as gateway:
         def fetch_url(input_url: str):