Pass parameters to the underlying htttp client (#520)

Author: martin-majlis

tests/http_client_kwargs_test.py

@@ -0,0 +1,298 @@
+"""Tests for HTTP client kwargs forwarding to httpx.
+
+This test module verifies that all kwargs passed to Wikipedia/AsyncWikipedia
+constructors are properly forwarded to underlying httpx client constructors.
+"""
+
+import httpx
+import pytest
+import respx
+
+from tests.mock_data import create_mock_async_wikipedia
+from tests.mock_data import create_mock_wikipedia
+
+
+class TestSyncHTTPClientKwargs:
+    """Test kwargs forwarding in SyncHTTPClient."""
+
+    @respx.mock
+    def test_timeout_parameter(self):
+        """Test that timeout parameter is forwarded to .Client."""
+        wiki = create_mock_wikipedia(timeout=30.0)
+        assert wiki._client.timeout == httpx.Timeout(30.0)
+
+    @respx.mock
+    def test_proxies_parameter(self):
+        """Test that proxy parameter is forwarded to .Client."""
+        proxy = "http://proxy.example.com:8080"
+        wiki = create_mock_wikipedia(proxy=proxy)
+        # httpx stores proxy info in the transport, not as a direct attribute
+        # We can verify the client was created successfully
+        assert wiki._client is not None
+        assert hasattr(wiki._client, "_transport")
+
+    @respx.mock
+    def test_verify_parameter_false(self):
+        """Test that verify=False is forwarded to .Client."""
+        wiki = create_mock_wikipedia(verify=False)
+        # httpx stores verify internally, but we can verify the client was created successfully
+        assert wiki._client is not None
+        # The fact that the client was created without error means verify was accepted
+
+    @respx.mock
+    def test_verify_parameter_custom(self):
+        """Test that custom verify context is forwarded to .Client."""
+        import ssl
+
+        custom_context = ssl.create_default_context()
+        wiki = create_mock_wikipedia(verify=custom_context)
+        # httpx stores verify internally, but we can verify the client was created successfully
+        assert wiki._client is not None
+        # The fact that the client was created without error means verify was accepted
+
+    @respx.mock
+    def test_http2_parameter(self):
+        """Test that http2 parameter is forwarded to .Client."""
+        try:
+            wiki = create_mock_wikipedia(http2=True)
+            assert wiki._client._http2 is True
+        except ImportError as e:
+            if "h2" in str(e):
+                pytest.skip("http2 not available (h2 package not installed)")
+            else:
+                raise
+
+    @respx.mock
+    def test_limits_parameter(self):
+        """Test that limits parameter is forwarded to .Client."""
+        limits = httpx.Limits(max_connections=50, max_keepalive_connections=10)
+        wiki = create_mock_wikipedia(limits=limits)
+        # httpx stores limits internally, but we can verify client was created successfully
+        assert wiki._client is not None
+        # The fact that client was created without error means limits was accepted
+
+    @respx.mock
+    def test_max_redirects_parameter(self):
+        """Test that max_redirects parameter is forwarded to .Client."""
+        wiki = create_mock_wikipedia(max_redirects=5)
+        assert wiki._client.max_redirects == 5
+
+    @respx.mock
+    def test_follow_redirects_parameter(self):
+        """Test that follow_redirects parameter is forwarded to .Client."""
+        wiki = create_mock_wikipedia(follow_redirects=True)
+        assert wiki._client.follow_redirects is True
+
+    @respx.mock
+    def test_auth_parameter(self):
+        """Test that auth parameter is forwarded to .Client."""
+        auth = httpx.BasicAuth("username", "password")
+        wiki = create_mock_wikipedia(auth=auth)
+        assert wiki._client._auth == auth
+
+    @respx.mock
+    def test_base_url_parameter(self):
+        """Test that base_url parameter is forwarded to .Client."""
+        base_url = "https://custom-api.example.com"
+        wiki = create_mock_wikipedia(base_url=base_url)
+        assert str(wiki._client._base_url) == base_url
+
+    @respx.mock
+    def test_default_encoding_parameter(self):
+        """Test that default_encoding parameter is forwarded to .Client."""
+        wiki = create_mock_wikipedia(default_encoding="latin-1")
+        assert wiki._client._default_encoding == "latin-1"
+
+    @respx.mock
+    def test_trust_env_parameter(self):
+        """Test that trust_env parameter is forwarded to .Client."""
+        wiki = create_mock_wikipedia(trust_env=False)
+        assert wiki._client._trust_env is False
+
+    @respx.mock
+    def test_combined_parameters(self):
+        """Test that multiple parameters work together."""
+        proxy = "http://proxy.example.com:8080"
+        limits = httpx.Limits(max_connections=50)
+
+        wiki = create_mock_wikipedia(
+            timeout=20.0,
+            proxy=proxy,
+            verify=False,
+            limits=limits,
+            max_redirects=3,
+            follow_redirects=True,
+        )
+
+        # Test the parameters that are accessible
+        assert wiki._client.timeout == httpx.Timeout(20.0)
+        assert wiki._client.max_redirects == 3
+        assert wiki._client.follow_redirects is True
+        # Verify the client was created successfully (means all parameters were accepted)
+        assert wiki._client is not None
+
+    @respx.mock
+    def test_headers_are_preserved(self):
+        """Test that custom headers are preserved when using kwargs."""
+        custom_headers = {"X-Custom-Header": "test-value"}
+        wiki = create_mock_wikipedia(headers=custom_headers, timeout=30.0)
+
+        # Check that our custom header is present
+        assert "X-Custom-Header" in wiki._client.headers
+        assert wiki._client.headers["X-Custom-Header"] == "test-value"
+
+        # Check that User-Agent is still set correctly
+        assert "User-Agent" in wiki._client.headers
+
+    @respx.mock
+    def test_invalid_httpx_parameter_raises_error(self):
+        """Test that invalid httpx parameters raise appropriate errors."""
+        # httpx should raise an error for invalid parameters
+        with pytest.raises(TypeError):
+            create_mock_wikipedia(invalid_param="should_fail")
+
+
+class TestAsyncHTTPClientKwargs:
+    """Test kwargs forwarding in AsyncHTTPClient."""
+
+    @respx.mock
+    async def test_timeout_parameter(self):
+        """Test that timeout parameter is forwarded to .AsyncClient."""
+        wiki = create_mock_async_wikipedia(timeout=30.0)
+        assert wiki._client.timeout == httpx.Timeout(30.0)
+
+    @respx.mock
+    async def test_proxies_parameter(self):
+        """Test that proxy parameter is forwarded to .AsyncClient."""
+        proxy = "http://proxy.example.com:8080"
+        wiki = create_mock_async_wikipedia(proxy=proxy)
+        # httpx stores proxy info in the transport, not as a direct attribute
+        # We can verify the client was created successfully
+        assert wiki._client is not None
+        assert hasattr(wiki._client, "_transport")
+
+    @respx.mock
+    async def test_verify_parameter_false(self):
+        """Test that verify=False is forwarded to .AsyncClient."""
+        wiki = create_mock_async_wikipedia(verify=False)
+        # httpx stores verify internally, but we can verify the client was created successfully
+        assert wiki._client is not None
+        # The fact that the client was created without error means verify was accepted
+
+    @respx.mock
+    async def test_http2_parameter(self):
+        """Test that http2 parameter is forwarded to .AsyncClient."""
+        try:
+            wiki = create_mock_async_wikipedia(http2=True)
+            assert wiki._client._http2 is True
+        except ImportError as e:
+            if "h2" in str(e):
+                pytest.skip("http2 not available (h2 package not installed)")
+            else:
+                raise
+
+    @respx.mock
+    async def test_limits_parameter(self):
+        """Test that limits parameter is forwarded to .AsyncClient."""
+        limits = httpx.Limits(max_connections=50, max_keepalive_connections=10)
+        wiki = create_mock_async_wikipedia(limits=limits)
+        # httpx stores limits internally, but we can verify client was created successfully
+        assert wiki._client is not None
+        # The fact that client was created without error means limits was accepted
+
+    @respx.mock
+    async def test_max_redirects_parameter(self):
+        """Test that max_redirects parameter is forwarded to .AsyncClient."""
+        wiki = create_mock_async_wikipedia(max_redirects=5)
+        assert wiki._client.max_redirects == 5
+
+    @respx.mock
+    async def test_follow_redirects_parameter(self):
+        """Test that follow_redirects parameter is forwarded to .AsyncClient."""
+        wiki = create_mock_async_wikipedia(follow_redirects=True)
+        assert wiki._client.follow_redirects is True
+
+    @respx.mock
+    async def test_auth_parameter(self):
+        """Test that auth parameter is forwarded to .AsyncClient."""
+        auth = httpx.BasicAuth("username", "password")
+        wiki = create_mock_async_wikipedia(auth=auth)
+        assert wiki._client._auth == auth
+
+    @respx.mock
+    async def test_base_url_parameter(self):
+        """Test that base_url parameter is forwarded to .AsyncClient."""
+        base_url = "https://custom-api.example.com"
+        wiki = create_mock_async_wikipedia(base_url=base_url)
+        assert str(wiki._client._base_url) == base_url
+
+    @respx.mock
+    async def test_combined_parameters(self):
+        """Test that multiple parameters work together in async client."""
+        proxy = "http://proxy.example.com:8080"
+        limits = httpx.Limits(max_connections=50)
+
+        wiki = create_mock_async_wikipedia(
+            timeout=20.0,
+            proxy=proxy,
+            verify=False,
+            limits=limits,
+            max_redirects=3,
+            follow_redirects=True,
+        )
+
+        # Test the parameters that are accessible
+        assert wiki._client.timeout == httpx.Timeout(20.0)
+        assert wiki._client.max_redirects == 3
+        assert wiki._client.follow_redirects is True
+        # Verify the client was created successfully (means all parameters were accepted)
+        assert wiki._client is not None
+
+    @respx.mock
+    async def test_headers_are_preserved(self):
+        """Test that custom headers are preserved when using kwargs in async client."""
+        custom_headers = {"X-Custom-Header": "test-value"}
+        wiki = create_mock_async_wikipedia(headers=custom_headers, timeout=30.0)
+
+        # Check that our custom header is present
+        assert "X-Custom-Header" in wiki._client.headers
+        assert wiki._client.headers["X-Custom-Header"] == "test-value"
+
+        # Check that User-Agent is still set correctly
+        assert "User-Agent" in wiki._client.headers
+
+    @respx.mock
+    async def test_invalid_httpx_parameter_raises_error(self):
+        """Test that invalid httpx parameters raise appropriate errors in async client."""
+        # httpx should raise an error for invalid parameters
+        with pytest.raises(TypeError):
+            create_mock_async_wikipedia(invalid_param="should_fail")
+
+
+class TestKwargsBackwardCompatibility:
+    """Test that existing usage patterns still work."""
+
+    @respx.mock
+    def test_backward_compatibility_timeout_only(self):
+        """Test that existing timeout-only usage still works."""
+        # This is how users currently use timeout
+        wiki = create_mock_wikipedia(timeout=25.0)
+        assert wiki._client.timeout == httpx.Timeout(25.0)
+
+    @respx.mock
+    async def test_backward_compatibility_timeout_only_async(self):
+        """Test that existing timeout-only usage still works in async client."""
+        wiki = create_mock_async_wikipedia(timeout=25.0)
+        assert wiki._client.timeout == httpx.Timeout(25.0)
+
+    @respx.mock
+    def test_default_timeout_still_works(self):
+        """Test that default timeout is still applied when not specified."""
+        wiki = create_mock_wikipedia()  # No timeout specified
+        assert wiki._client.timeout == httpx.Timeout(10.0)  # Should be default
+
+    @respx.mock
+    async def test_default_timeout_still_works_async(self):
+        """Test that default timeout is still applied when not specified in async client."""
+        wiki = create_mock_async_wikipedia()  # No timeout specified
+        assert wiki._client.timeout == httpx.Timeout(10.0)  # Should be default

tests/mock_data.py

@@ -1006,7 +1006,7 @@ async def api_request(language, params):
 # ── CLI-specific mock data and helpers ───────────────────────────────────────────
 
 
-def create_mock_wikipedia(language="en", variant=None, extract_format="wiki"):
+def create_mock_wikipedia(language="en", variant=None, extract_format="wiki", **kwargs):
     """Create a mock Wikipedia instance for testing CLI functions."""
     import wikipediaapi
 
@@ -1019,11 +1019,31 @@ def create_mock_wikipedia(language="en", variant=None, extract_format="wiki"):
             if extract_format == "wiki"
             else wikipediaapi.ExtractFormat.HTML
         ),
+        **kwargs,
     )
     wiki._get = wikipedia_api_request(wiki)
     return wiki
 
 
+def create_mock_async_wikipedia(language="en", variant=None, extract_format="wiki", **kwargs):
+    """Create a mock AsyncWikipedia instance for testing."""
+    import wikipediaapi
+
+    wiki = wikipediaapi.AsyncWikipedia(
+        user_agent=user_agent,
+        language=language,
+        variant=variant,
+        extract_format=(
+            wikipediaapi.ExtractFormat.WIKI
+            if extract_format == "wiki"
+            else wikipediaapi.ExtractFormat.HTML
+        ),
+        **kwargs,
+    )
+    wiki._get = async_wikipedia_api_request(wiki)
+    return wiki
+
+
 def create_mock_page(title, pageid=1, exists=True, language="en", namespace=0):
     """Create a mock Wikipedia page for testing."""
     import wikipediaapi

wikipediaapi/_http_client/async_http_client.py

@@ -50,7 +50,7 @@ def __init__(self, *args: Any, **kwargs: Any) -> None:
         super().__init__(*args, **kwargs)
         self._client = httpx.AsyncClient(
             headers=self._default_headers,
-            timeout=self._client_kwargs.get("timeout", 10.0),
+            **self._client_kwargs,
             transport=httpx.AsyncHTTPTransport(),
         )
 

wikipediaapi/_http_client/base_http_client.py

@@ -43,6 +43,13 @@ class BaseHTTPClient(ABC):
     Subclasses must implement ``_get(language, params)`` (sync or async)
     that issues the actual HTTP request.
 
+    **Note on HTTP Library**: This implementation uses ``httpx`` as the underlying
+    HTTP client library.  Advanced HTTP configuration (timeouts, proxies,
+    SSL settings, connection limits, etc.) is exposed through the
+    ``SyncHTTPClient`` and ``AsyncHTTPClient`` classes.  For most use
+    cases, use the standard Wikipedia API parameters.  Direct httpx
+    configuration should only be needed for advanced use cases.
+
     Instance attributes set by :meth:`__init__`:
 
     :attr language: normalised Wikipedia language code (e.g. ``"en"``)
@@ -87,7 +94,13 @@ def __init__(
         :param retry_wait: base wait time in seconds between retries
             (exponential backoff); overridden by ``Retry-After`` for 429
         :param kwargs: forwarded to ``httpx`` client constructor
-            (e.g. ``timeout=30.0``); ``timeout`` defaults to ``10.0``
+            (e.g. ``timeout=30.0``, ``proxy={'https://': 'http://proxy.example.com:8080'}``,
+            ``verify=False``, ``http2=True``); ``timeout`` defaults to ``10.0``.
+            **Advanced Usage**: These parameters provide direct access to httpx
+            capabilities.  For standard Wikipedia API usage, prefer the
+            documented parameters above.  Use httpx parameters only for
+            specific requirements like custom proxies, SSL configuration, or
+            connection pooling.
         :raises AssertionError: if *user_agent* is too short or
             *language* is empty
         """

wikipediaapi/_http_client/sync_http_client.py

@@ -48,7 +48,7 @@ def __init__(self, *args: Any, **kwargs: Any) -> None:
         super().__init__(*args, **kwargs)
         self._client = httpx.Client(
             headers=self._default_headers,
-            timeout=self._client_kwargs.get("timeout", 10.0),
+            **self._client_kwargs,
             transport=httpx.HTTPTransport(),
         )
 

wikipediaapi/async_wikipedia.py

@@ -65,7 +65,11 @@ async def main():
         ``Retry-After`` header value is used instead.  Defaults to
         ``1.0``.
     :param kwargs: additional keyword arguments forwarded to
-        ``httpx.AsyncClient`` (e.g. ``timeout=30.0``).
+        ``httpx.AsyncClient`` (e.g. ``timeout=30.0``, ``proxies={…}``,
+        ``verify=False``).  **Advanced Usage**: These provide direct
+        access to httpx capabilities.  For most use cases, prefer the
+        standard parameters above.  Use httpx parameters only for specific
+        requirements like custom proxies, SSL configuration, or connection pooling.
     """
 
     pass