Added envvar proxy support to http gateway and minor improvements.

Author: theypsilon

src/downloader/config.py

@@ -25,6 +25,7 @@
     DEFAULT_MINIMUM_SYSTEM_FREE_SPACE_MB, DEFAULT_MINIMUM_EXTERNAL_FREE_SPACE_MB
 from downloader.db_options import DbOptions
 from downloader.error import DownloaderError
+from downloader.http_gateway import HttpConfig
 
 
 class Environment(TypedDict):
@@ -43,6 +44,8 @@ class Environment(TypedDict):
     PC_LAUNCHER: Optional[str]
     DEBUG: str
     FAIL_ON_FILE_ERROR: str
+    HTTP_PROXY: str
+    HTTPS_PROXY: str
 
 
 @unique
@@ -85,7 +88,7 @@ class ConfigMisterSection(TypedDict):
     user_defined_options: List[str]
 
 
-class ConfigRequired(ConfigMisterSection):
+class ConfigRequired(ConfigMisterSection, HttpConfig):
     zip_file_count_threshold: int
     zip_accumulated_mb_threshold: int
     debug: bool
@@ -138,7 +141,9 @@ def default_config() -> Config:
         'commit': 'unknown',
         'fail_on_file_error': False,
         'minimum_system_free_space_mb': DEFAULT_MINIMUM_SYSTEM_FREE_SPACE_MB,
-        'minimum_external_free_space_mb': DEFAULT_MINIMUM_EXTERNAL_FREE_SPACE_MB
+        'minimum_external_free_space_mb': DEFAULT_MINIMUM_EXTERNAL_FREE_SPACE_MB,
+        'http_proxy': None,
+        'https_proxy': None
     }
 
 

src/downloader/config_reader.py

@@ -32,6 +32,7 @@
     K_MINIMUM_EXTERNAL_FREE_SPACE_MB, STORAGE_PRIORITY_OFF, STORAGE_PRIORITY_PREFER_SD, \
     STORAGE_PRIORITY_PREFER_EXTERNAL, EXIT_ERROR_WRONG_SETUP
 from downloader.db_options import DbOptions, DbOptionsProps, DbOptionsValidationException
+from downloader.http_gateway import http_config
 from downloader.logger import Logger, time_str
 
 
@@ -158,6 +159,9 @@ def read_config(self, config_path: str) -> Config:
             result['logfile'] = str(launcher_path.with_suffix('.log'))
             result['curl_ssl'] = ''
 
+        if self._env['HTTP_PROXY'] or self._env['HTTPS_PROXY']:
+            result.update(http_config(http_proxy=self._env['HTTP_PROXY'], https_proxy=self._env['HTTPS_PROXY']))
+
         result['environment'] = self._env
 
         if result['verbose']: self._logger.print(f'BENCH {time_str(self._start_time)}| Read config done.')

src/downloader/constants.py

@@ -180,6 +180,10 @@ def FILE_7z_util_uninstalled_description() -> SafeFetchInfo: return {
 KENV_FAIL_ON_FILE_ERROR: Final[str] = 'FAIL_ON_FILE_ERROR'
 KENV_LOGFILE: Final[str] = 'LOGFILE'
 KENV_LOGLEVEL: Final[str] = 'LOGLEVEL'
+KENV_HTTP_PROXY: Final[str] = 'HTTP_PROXY'
+KENV_LC_HTTP_PROXY: Final[str] = 'http_proxy'
+KENV_HTTPS_PROXY: Final[str] = 'HTTPS_PROXY'
+KENV_LC_HTTPS_PROXY: Final[str] = 'https_proxy'
 
 # Db State Signature
 DB_STATE_SIGNATURE_NO_HASH: Final[str] = 'non_initialized_hash'

src/downloader/full_run_service_factory.py

@@ -77,7 +77,8 @@ def create(self, config: Config):
         http_gateway = HttpGateway(
             ssl_ctx=ssl_ctx,
             timeout=http_connection_timeout,
-            logger=DebugOnlyLoggerDecorator(self._logger) if config['http_logging'] else None
+            logger=DebugOnlyLoggerDecorator(self._logger) if config['http_logging'] else None,
+            config=config
         )
         atexit.register(http_gateway.cleanup)
         safe_file_fetcher = SafeFileFetcher(config, system_file_system, self._logger, http_gateway, waiter)

src/downloader/http_gateway.py

@@ -22,7 +22,7 @@
 import time
 from contextlib import contextmanager
 from email.utils import parsedate_to_datetime
-from typing import Type, Tuple, Any, Optional, Generator, List, Dict, Union, Protocol, TypeVar, Generic
+from typing import Type, Tuple, Any, Optional, Generator, List, Dict, Union, Protocol, TypeVar, Generic, TypedDict
 from urllib.parse import urlparse, ParseResult, urlunparse
 from http.client import HTTPConnection, HTTPSConnection, HTTPResponse, HTTPException
 from types import TracebackType
@@ -36,12 +36,18 @@ def print(self, *args: Any) -> None: ...
     def debug(self, *args: Any) -> None: ...
 
 
+class HttpConfig(TypedDict):
+    http_proxy: Optional[tuple[str, str, int]]  # (scheme, host, port)
+    https_proxy: Optional[tuple[str, str, int]]  # (scheme, host, port)
+
+
 class HttpGateway:
-    def __init__(self, ssl_ctx: ssl.SSLContext, timeout: float, logger: Optional[HttpLogger] = None) -> None:
+    def __init__(self, ssl_ctx: ssl.SSLContext, timeout: float, logger: Optional[HttpLogger] = None, config: Optional[dict[str, Any]] = None) -> None:
         now = time.monotonic()
         self._ssl_ctx = ssl_ctx
         self._timeout = timeout
         self._logger = logger
+        self._config = config
         self._connections: Dict[_QueueId, _ConnectionQueue] = {}
         self._connections_lock = threading.Lock()
         self._clean_timeout_connections_timer = now
@@ -79,7 +85,7 @@ def open(self, url: str, method: Optional[str] = None, body: Any = None, headers
             parsed_url,
             method,
             body,
-            headers or _default_headers,
+            {**_default_headers, **headers} if isinstance(headers, dict) else _default_headers,
         )
         if self._logger is not None: self._logger.debug(f'HTTP {conn.response.status}: {final_url}\n'
                                                         f'1st byte @ {time.monotonic() - now:.3f}s\nvvvv\n')
@@ -166,7 +172,7 @@ def _process_queue_id(self, queue_id: '_QueueId') -> '_QueueId': return _redirec
     def _take_connection(self, queue_id: '_QueueId') -> '_Connection':
         with self._connections_lock:
             if queue_id not in self._connections:
-                self._connections[queue_id] = _ConnectionQueue(queue_id, self._timeout, self._ssl_ctx, self._logger)
+                self._connections[queue_id] = _ConnectionQueue(queue_id, self._timeout, self._ssl_ctx, self._logger, self._config)
             return self._connections[queue_id].pull()
 
     def _clean_timeout_connections(self, now: float) -> None:
@@ -238,7 +244,26 @@ def _fill_redirects_swap(self, now: float, lock: threading.Lock, redirects: Dict
 
         return size != len(self._redirects_swap)
 
-_default_headers = {'Connection': 'keep-alive', 'Keep-Alive': 'timeout=120'}
+
+def http_config(http_proxy: Optional[str], https_proxy: Optional[str]) -> HttpConfig:
+    config: HttpConfig = {"http_proxy": None, "https_proxy": None}
+    if http_proxy:
+        parsed = urlparse(http_proxy)
+        if parsed.hostname and parsed.scheme in ('http', 'https'):
+            config['http_proxy'] = (parsed.scheme, parsed.hostname, parsed.port or (443 if parsed.scheme == 'https' else 80))
+
+    if not https_proxy and http_proxy:
+        https_proxy = http_proxy
+
+    if https_proxy:
+        parsed = urlparse(https_proxy)
+        if parsed.hostname and parsed.scheme in ('http', 'https'):
+            config['https_proxy'] = (parsed.scheme, parsed.hostname, parsed.port or (443 if parsed.scheme == 'https' else 80))
+
+    return config
+
+USER_AGENT = 'Downloader/2.X (Linux; [email protected])'
+_default_headers = {'User-Agent': USER_AGENT, 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=120'}
 
 
 _QueueId = Tuple[str, str]
@@ -274,7 +299,7 @@ def __init__(self, conn_id: int, http: HTTPConnection, connection_queue: '_Conne
         self._timeout: float = http.timeout if http.timeout is not None else 120.0
         self._last_use_time: float = 0.0
         self._uses: int = 0
-        self._max_uses: float = sys.float_info.max
+        self._max_uses: int = sys.maxsize
         self._response: Optional[Union[HTTPResponse, '_FinishedResponse']] = None
         self._response_headers = _ResponseHeaders(logger)
 
@@ -306,8 +331,11 @@ def response_headers(self) -> '_ResponseHeaders':
         return self._response_headers
 
     def finish_response(self) -> None:
-        if self._close_response() and self._uses < self._max_uses:
-            self._connection_queue.push(self)
+        if self._close_response():
+            if self._uses < self._max_uses:
+                self._connection_queue.push(self)
+            else:
+                self._http.close()
 
     def _close_response(self) -> bool:
         if isinstance(self._response, _FinishedResponse):
@@ -340,11 +368,12 @@ class _FinishedResponse: pass
 
 
 class _ConnectionQueue:
-    def __init__(self, queue_id: _QueueId, timeout: float, ctx: ssl.SSLContext, logger: Optional[HttpLogger]) -> None:
+    def __init__(self, queue_id: _QueueId, timeout: float, ctx: ssl.SSLContext, logger: Optional[HttpLogger], config: Optional[dict[str, Any]]) -> None:
         self.id = queue_id
         self._timeout = timeout
         self._ctx = ctx
         self._logger = logger
+        self._config = config
         self._queue: List[_Connection] = []
         self._queue_swap: List[_Connection] = []
         self._lock = threading.Lock()
@@ -354,7 +383,7 @@ def pull(self) -> _Connection:
         with self._lock:
             if len(self._queue) == 0:
                 self._last_conn_id += 1
-                http_conn = create_http_connection(self.id[0], self.id[1], self._timeout, self._ctx)
+                http_conn = create_http_connection(self.id[0], self.id[1], self._timeout, self._ctx, self._config)
                 return _Connection(conn_id=self._last_conn_id, http=http_conn, connection_queue=self, logger=self._logger)
             return self._queue.pop()
 
@@ -388,10 +417,31 @@ def clear_timed_outs(self, now: float) -> int:
             return expired_count
 
 
-def create_http_connection(scheme: str, netloc: str, timeout: float, ctx: ssl.SSLContext) -> HTTPConnection:
-    if scheme == 'http': return HTTPConnection(netloc, timeout=timeout)
-    elif scheme == 'https': return HTTPSConnection(netloc, timeout=timeout, context=ctx)
-    else: raise HttpGatewayException(f"Scheme {scheme} not supported")
+def create_http_connection(scheme: str, netloc: str, timeout: float, ctx: ssl.SSLContext, config: Optional[dict[str, Any]]) -> HTTPConnection:
+    if scheme == 'http':
+        if config and config['http_proxy']:
+            proxy_scheme, proxy_host, proxy_port = config['http_proxy']
+            if proxy_scheme == 'https':
+                return HTTPSConnection(proxy_host, proxy_port, timeout=timeout, context=ctx)
+            return HTTPConnection(proxy_host, proxy_port, timeout=timeout)
+        return HTTPConnection(netloc, timeout=timeout)
+
+    elif scheme == 'https':
+        if config and config['https_proxy']:
+            proxy_scheme, proxy_host, proxy_port = config['https_proxy']
+            parsed_netloc = urlparse(f'//{netloc}')
+            target_host = parsed_netloc.hostname
+            target_port = parsed_netloc.port or 443
+            if not target_host:
+                raise HttpGatewayException(f"Invalid netloc: {netloc}")
+
+            conn = HTTPSConnection(proxy_host, proxy_port, timeout=timeout, context=ctx)
+            conn.set_tunnel(target_host, target_port)
+            return conn
+        return HTTPSConnection(netloc, timeout=timeout, context=ctx)
+
+    else:
+        raise HttpGatewayException(f"Scheme {scheme} not supported")
 
 
 class _ResponseHeaders:
@@ -426,16 +476,15 @@ def redirect_params(self, status: int) -> Tuple[Optional[str], Optional[float]]:
                     return new_url, None
 
                 age = self._headers.get('age', 0)
-                try:
-                    age = int(age)
-                except Exception as e:
-                    if self._logger is not None: self._logger.debug(f"Could not parse Age from {age}", e)
-                    age = 0
+                if age != 0:
+                    try:
+                        age = int(age)
+                    except Exception as e:
+                        if self._logger is not None: self._logger.debug(f"Could not parse Age from {age}", e)
+                        age = 0
 
                 return new_url, time.monotonic() + max_age - age
 
-            pass
-
         expires = self._headers.get('expires', None)
         if expires is not None:
             try:

src/downloader/main.py

@@ -26,7 +26,8 @@
 
 from downloader.config import Environment
 from downloader.config_reader import ConfigReader
-from downloader.constants import KENV_LOGLEVEL
+from downloader.constants import KENV_LOGLEVEL, KENV_LC_HTTP_PROXY, KENV_HTTP_PROXY, KENV_HTTPS_PROXY, \
+    KENV_LC_HTTPS_PROXY
 from downloader.logger import TopLogger
 from downloader.full_run_service_factory import FullRunServiceFactory
 
@@ -75,6 +76,8 @@ def read_env(default_commit: Optional[str]) -> Environment:
         'PC_LAUNCHER': os.getenv(KENV_PC_LAUNCHER, None),
         'DEBUG': os.getenv(KENV_DEBUG, 'false').lower(),
         'FAIL_ON_FILE_ERROR': os.getenv(KENV_FAIL_ON_FILE_ERROR, 'false'),
+        'HTTP_PROXY': os.getenv(KENV_HTTP_PROXY) or os.getenv(KENV_LC_HTTP_PROXY),
+        'HTTPS_PROXY': os.getenv(KENV_HTTPS_PROXY) or os.getenv(KENV_LC_HTTPS_PROXY)
     }
 
 

src/test/exploratory/http_gateway_connections/explore_http_gateway_with_real_urls.py

@@ -27,7 +27,7 @@
 from pathlib import Path
 from typing import List
 
-from downloader.http_gateway import HttpGateway, HttpLogger
+from downloader.http_gateway import HttpGateway, HttpLogger, http_config
 
 urls = [
     'https://google.com',
@@ -69,7 +69,27 @@ def main() -> None:
     cancelled = 0
     dir_path = f'{os.path.dirname(os.path.realpath(__file__))}/delme'
 
-    with HttpGateway(ssl_ctx=ssl.create_default_context(), timeout=180, logger=logger) as gateway:
+    http_proxy_url = os.environ.get('HTTP_PROXY')
+    https_proxy_url = os.environ.get('HTTPS_PROXY')
+    config = http_config(http_proxy=http_proxy_url, https_proxy=https_proxy_url) if (http_proxy_url or https_proxy_url) else None
+
+    if config:
+        logger.print('PROXY CONFIGURATION:')
+        if config.get('http_proxy'):
+            scheme, host, port = config['http_proxy']
+            logger.print(f'  HTTP Proxy:  {scheme}://{host}:{port}')
+        else:
+            logger.print(f'  HTTP Proxy:  None (direct connection)')
+
+        if config.get('https_proxy'):
+            scheme, host, port = config['https_proxy']
+            logger.print(f'  HTTPS Proxy: {scheme}://{host}:{port}')
+        else:
+            logger.print(f'  HTTPS Proxy: None (direct connection)')
+    else:
+        logger.print('NO PROXY - Using direct connection')
+
+    with HttpGateway(ssl_ctx=ssl.create_default_context(), timeout=180, logger=logger, config=config) as gateway:
         def fetch_url(input_url: str):
             nonlocal interrupted, gateway, dir_path
             with gateway.open(input_url) as (url, res):

src/test/objects.py

@@ -24,7 +24,7 @@
     K_FILTER, KENV_DEFAULT_DB_URL, KENV_DEFAULT_DB_ID, KENV_DEFAULT_BASE_PATH, KENV_ALLOW_REBOOT, KENV_DEBUG, MEDIA_FAT, MEDIA_USB0, MEDIA_USB1, \
     MEDIA_USB2, KENV_FAIL_ON_FILE_ERROR, KENV_UPDATE_LINUX, KENV_CURL_SSL, KENV_COMMIT, DEFAULT_CURL_SSL_OPTIONS, \
     MEDIA_USB3, KENV_LOGFILE, KENV_PC_LAUNCHER, DEFAULT_UPDATE_LINUX_ENV, K_DB_URL, K_SECTION, K_OPTIONS, KENV_FORCED_BASE_PATH, \
-    FILE_MiSTer_old
+    FILE_MiSTer_old, KENV_HTTP_PROXY, KENV_HTTPS_PROXY
 from downloader.db_options import DbOptions
 from downloader.other import empty_store_without_base_path
 from downloader.db_entity import DbEntity
@@ -150,6 +150,8 @@ def config_with(
         default_db_id=None,
         user_defined_options=None,
         minimum_free_space=None,
+        file_checking=None,
+
         databases: Dict[str, Any] = None):
 
     config = default_config()
@@ -177,6 +179,8 @@ def config_with(
         config['user_defined_options'] = user_defined_options
     if minimum_free_space is not None:
         config['minimum_system_free_space_mb'] = minimum_free_space
+    if file_checking is not None:
+        config['file_checking'] = file_checking
     return config
 
 
@@ -782,6 +786,8 @@ def default_env() -> Environment:
         KENV_LOGFILE: None,
         KENV_LOGLEVEL: '',
         KENV_PC_LAUNCHER: None,
+        KENV_HTTP_PROXY: None,
+        KENV_HTTPS_PROXY: None
     }