From 63abceedead7a7cb2e2153cc7dba59d60f3a7e9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jind=C5=99ich=20B=C3=A4r?= Date: Sun, 9 Nov 2025 09:07:47 +0100 Subject: [PATCH 01/13] chore: add anonymous client API tests --- impit-python/test/no_client_test.py | 411 ++++++++++++++++++++++++++++ 1 file changed, 411 insertions(+) create mode 100644 impit-python/test/no_client_test.py diff --git a/impit-python/test/no_client_test.py b/impit-python/test/no_client_test.py new file mode 100644 index 00000000..ca446db6 --- /dev/null +++ b/impit-python/test/no_client_test.py @@ -0,0 +1,411 @@ +import json +import socket +import threading +import time +from http.cookiejar import CookieJar + +import pytest + +from impit import Browser, Client, Cookies, StreamClosed, StreamConsumed, TooManyRedirects +import impit + +from .httpbin import get_httpbin_url + + +def thread_server(port_holder: list[int]) -> None: + server = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) + server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + server.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 0) # allow IPv4/IPv6 in Windows + server.bind(('::', 0)) + port_holder[0] = server.getsockname()[1] + server.listen(1) + + conn, addr = server.accept() + conn.recv(1024) + client_ip, *_ = addr + response = f'HTTP/1.1 200 OK\r\nContent-Length: {len(client_ip)}\r\n\r\n{client_ip}'.encode() + conn.send(response) + conn.close() + server.close() + +class TestBasicRequests: + @pytest.mark.parametrize( + ('protocol'), + ['http://', 'https://'], + ) + def test_basic_requests(self, protocol: str) -> None: + resp = impit.get(f'{protocol}apify.com') + assert resp.status_code == 200 if protocol == 'https://' else resp.status_code == 301 + + def test_boringssl_based_server(self) -> None: + response = impit.get('https://www.google.com') + assert response.status_code == 200 + assert response.text + + def test_content_encoding(self) -> None: + resp = impit.get(get_httpbin_url('/encoding/utf8')) + assert resp.status_code == 200 + assert resp.encoding == 'utf-8' + + def test_headers_work(self) -> None: + response = impit.get(get_httpbin_url('/headers'), headers={'Impit-Test': 'foo'}) + assert response.status_code == 200 + assert response.json()['headers']['Impit-Test'] == 'foo' + + def test_cookies_nonstandard(self) -> None: + cookies_jar = CookieJar() + impit.get(get_httpbin_url('/cookies/set', query={'set-by-server': '321'}), cookie_jar=cookies_jar, follow_redirects=True) + + for cookie in cookies_jar: + assert cookie.has_nonstandard_attr('HttpOnly') is not None + + def test_complex_cookies(self) -> None: + cookies_jar = CookieJar() + + url = get_httpbin_url( + '/response-headers', + query={ + 'set-cookie': [ + 'basic=1; Path=/; HttpOnly; SameSite=Lax', + 'withpath=2; Path=/html; SameSite=None', + 'strict=3; Path=/; SameSite=Strict', + 'secure=4; Path=/; HttpOnly; Secure; SameSite=Strict', + 'short=5; Path=/;', + 'domain=6; Path=/; Domain=.127.0.0.1;', + ] + }, + ) + + impit.get(url, cookie_jar=cookies_jar, follow_redirects=True) + + assert len(cookies_jar) == 6 + for cookie in cookies_jar: + if cookie.name == 'basic': + assert cookie.value == '1' + assert cookie.secure is False + assert cookie.has_nonstandard_attr('HttpOnly') is True + assert cookie.get_nonstandard_attr('SameSite') == 'Lax' + elif cookie.name == 'withpath': + assert cookie.value == '2' + assert cookie.secure is False + assert cookie.get_nonstandard_attr('SameSite') == 'None' + assert cookie.has_nonstandard_attr('HttpOnly') is False + assert cookie.path == '/html' + elif cookie.name == 'strict': + assert cookie.value == '3' + assert cookie.secure is False + assert cookie.has_nonstandard_attr('HttpOnly') is False + assert cookie.get_nonstandard_attr('SameSite') == 'Strict' + elif cookie.name == 'secure': + assert cookie.value == '4' + assert cookie.secure is True + assert cookie.has_nonstandard_attr('HttpOnly') is True + assert cookie.get_nonstandard_attr('SameSite') == 'Strict' + elif cookie.name == 'short': + assert cookie.value == '5' + assert cookie.secure is False + assert cookie.has_nonstandard_attr('SameSite') is False + elif cookie.name == 'domain': + assert cookie.value == '6' + assert cookie.secure is False + # Crate cookies, ignores the starting dot in the domain + # but it's ok - https://www.rfc-editor.org/rfc/rfc6265#section-4.1.2.3 + assert cookie.domain == '127.0.0.1' + + def test_cookie_jar_works(self) -> None: + cookies = Cookies({'preset-cookie': '123'}) + + response = impit.get( + get_httpbin_url('/cookies/'), + cookie_jar=cookies.jar, + ).json() + + assert response['cookies'] == {'preset-cookie': '123'} + + impit.get( + get_httpbin_url('/cookies/set', query={'set-by-server': '321'}), + ) + + response = impit.get( + get_httpbin_url('/cookies/'), + ).json() + + assert response['cookies'] == { + 'preset-cookie': '123', + 'set-by-server': '321', + } + + assert len(cookies.jar) == 2 + + def test_cookies_param_works(self, browser: Browser) -> None: + cookies = Cookies({'preset-cookie': '123'}) + + response = impit.get( + get_httpbin_url('/cookies/'), + cookies=cookies, + ).json() + + assert response['cookies'] == {'preset-cookie': '123'} + + impit.get( + get_httpbin_url('/cookies/set', query={'set-by-server': '321'}), + ) + + response = impit.get( + get_httpbin_url('/cookies/'), + ).json() + + assert response['cookies'] == { + 'preset-cookie': '123', + 'set-by-server': '321', + } + + assert len(cookies) == 2 + assert cookies.get('preset-cookie') == '123' + assert cookies.get('set-by-server') == '321' + + @pytest.mark.skip(reason='Flaky under the CI environment') + def test_http3_works(self) -> None: + response = impit.get('https://curl.se', force_http3=True) + assert response.status_code == 200 + assert 'curl' in response.text + assert response.http_version == 'HTTP/3' + + @pytest.mark.parametrize( + ('method'), + ['GET', 'POST', 'PUT', 'DELETE', 'PATCH', 'HEAD', 'OPTIONS'], + ) + def test_methods_work(self, method: str) -> None: + m = getattr(impit, method.lower()) + m(get_httpbin_url('/anything')) + + def test_default_no_redirect(self) -> None: + target_url = 'https://crawlee.dev/' + redirect_url = get_httpbin_url('/redirect-to', query={'url': target_url}) + + response = impit.get(redirect_url) + + assert response.status_code == 302 + assert response.is_redirect + + assert response.url == redirect_url + assert response.headers.get('location') == target_url + + def test_follow_redirects(self) -> None: + target_url = 'https://crawlee.dev/' + redirect_url = get_httpbin_url('/redirect-to', query={'url': target_url}) + + response = impit.get(redirect_url, follow_redirects=True) + + assert response.status_code == 200 + assert not response.is_redirect + + assert response.url == target_url + + def test_limit_redirects(self) -> None: + redirect_url = get_httpbin_url('/absolute-redirect/3') + + with pytest.raises(TooManyRedirects): + impit.get(redirect_url, follow_redirects=True, max_redirects=1) + + def test_thread_server(self) -> None: + port_holder = [0] + thread = threading.Thread(target=thread_server, args=(port_holder,)) + thread.start() + time.sleep(0.1) + + response = impit.get(f'http://127.0.0.1:{port_holder[0]}/', timeout=5) + assert response.status_code == 200 + thread.join() + + @pytest.mark.parametrize('addresses', [['127.0.0.1', '::ffff:127.0.0.1'], ['::1', '::1']]) + def test_local_address(self, browser: Browser, addresses: tuple[str, str]) -> None: + port_holder = [0] + thread = threading.Thread(target=thread_server, args=(port_holder,)) + thread.start() + time.sleep(0.1) + + [local_address, remote_address] = addresses + + response = impit.get(f'http://localhost:{port_holder[0]}/', timeout=5, local_address=local_address) + assert response.text == remote_address + assert response.status_code == 200 + thread.join() + + +class TestRequestBody: + def test_passing_string_body(self) -> None: + response = impit.post( + get_httpbin_url('/post'), + content=bytearray('{"Impit-Test":"foořžš"}', 'utf-8'), + headers={'Content-Type': 'application/json'}, + ) + assert response.status_code == 200 + assert response.json()['data'] == '{"Impit-Test":"foořžš"}' + + def test_passing_string_body_in_data(self) -> None: + response = impit.post( + get_httpbin_url('/post'), + data=bytearray('{"Impit-Test":"foořžš"}', 'utf-8'), # type: ignore[arg-type] + headers={'Content-Type': 'application/json'}, + ) + assert response.status_code == 200 + assert response.json()['data'] == '{"Impit-Test":"foořžš"}' + + def test_form_non_ascii(self) -> None: + response = impit.post( + get_httpbin_url('/post'), + data={'Impit-Test': '👾🕵🏻‍♂️🧑‍💻'}, + ) + assert response.status_code == 200 + assert response.json()['form']['Impit-Test'] == '👾🕵🏻‍♂️🧑‍💻' + + def test_passing_binary_body(self) -> None: + response = impit.post( + get_httpbin_url('/post'), + content=[ + 0x49, + 0x6D, + 0x70, + 0x69, + 0x74, + 0x2D, + 0x54, + 0x65, + 0x73, + 0x74, + 0x3A, + 0x66, + 0x6F, + 0x6F, + 0xC5, + 0x99, + 0xC5, + 0xBE, + 0xC5, + 0xA1, + ], + headers={'Content-Type': 'application/json'}, + ) + assert response.status_code == 200 + assert response.json()['data'] == 'Impit-Test:foořžš' + + @pytest.mark.parametrize( + ('method'), + ['POST', 'PUT', 'PATCH'], + ) + def test_methods_accept_request_body(self, method: str) -> None: + m = getattr(impit, method.lower()) + + response = m(get_httpbin_url(f'/{method.lower()}'), content=b'foo') + assert response.status_code == 200 + assert response.json()['data'] == 'foo' + + def test_content(self) -> None: + response = impit.get(get_httpbin_url('/')) + + assert response.status_code == 200 + assert isinstance(response.content, bytes) + assert isinstance(response.text, str) + assert response.content.decode('utf-8') == response.text + + def test_json(self) -> None: + response = impit.get(get_httpbin_url('/get')) + + assert response.status_code == 200 + assert response.json() == json.loads(response.text) + +class TestStreamRequest: + def test_read(self) -> None: + with impit.stream('GET', get_httpbin_url('/')) as response: + assert response.status_code == 200 + assert response.is_closed is False + assert response.is_stream_consumed is False + + content = response.read() + + assert isinstance(content, bytes) + assert content.decode('utf-8') == response.text + assert response.content == content + + assert response.is_closed is True + assert response.is_stream_consumed is True # type: ignore[unreachable] # Mypy can't detect a change of state + + def test_iter_bytes(self) -> None: + with impit.stream('GET', get_httpbin_url('/')) as response: + assert response.status_code == 200 + assert response.is_closed is False + assert response.is_stream_consumed is False + + content = b''.join(response.iter_bytes()) + + assert isinstance(content, bytes) + assert len(content) > 0 + + # After `iter_bytes`` we should get an error since `content` and `text` are not cached + with pytest.raises(StreamConsumed): + _ = response.text + + with pytest.raises(StreamConsumed): + _ = response.content + + assert response.is_closed is True + assert response.is_stream_consumed is True # type: ignore[unreachable] # Mypy can't detect a change of state + + def test_response_with_context_manager(self) -> None: + with impit.stream('GET', get_httpbin_url('/')) as response: + assert response.status_code == 200 + assert response.is_closed is False + assert response.is_stream_consumed is False + + assert response.is_closed is True + assert response.is_stream_consumed is False # type: ignore[unreachable] # Mypy can't detect a change of state + + def test_read_after_close(self) -> None: + with impit.stream('GET', get_httpbin_url('/')) as response: + assert response.status_code == 200 + + assert response.is_closed is True + + with pytest.raises(StreamClosed): + _ = response.read() + + def test_two_read_calls(self) -> None: + with impit.stream('GET', get_httpbin_url('/')) as response: + assert response.status_code == 200 + + content = response.read() + assert isinstance(content, bytes) + assert content == response.content + + # Return content from cache + assert response.read() == response.content + + def test_two_iter_bytes_calls(self) -> None: + with impit.stream('GET', get_httpbin_url('/')) as response: + assert response.status_code == 200 + + content = b''.join(response.iter_bytes()) + assert isinstance(content, bytes) + assert len(content) > 0 + + # `iter_bytes` don't cache content + with pytest.raises(StreamConsumed): + _ = b''.join(response.iter_bytes()) + + def test_iter_bytes_without_consumed(self) -> None: + with impit.stream('GET', get_httpbin_url('/')) as response: + assert response.status_code == 200 + + iterator = response.iter_bytes() + + _ = next(iterator) + + assert response.is_closed is True + assert response.is_stream_consumed is False + + with pytest.raises(StreamClosed): + _ = response.text + + with pytest.raises(StreamClosed): + _ = response.content From 87298623325ff1c0087a8f003586abcd1c39d9b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jind=C5=99ich=20B=C3=A4r?= Date: Sun, 9 Nov 2025 09:15:48 +0100 Subject: [PATCH 02/13] fix(py): add `follow_redirects` option to the anonymous client API --- impit-python/src/lib.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/impit-python/src/lib.rs b/impit-python/src/lib.rs index 8ff82e21..d02ca268 100644 --- a/impit-python/src/lib.rs +++ b/impit-python/src/lib.rs @@ -87,7 +87,7 @@ fn impit(_py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> { ($($name:ident),*) => { $( #[pyfunction] - #[pyo3(signature = (url, content=None, data=None, headers=None, timeout=None, force_http3=false, cookie_jar=None, cookies=None))] + #[pyo3(signature = (url, content=None, data=None, headers=None, timeout=None, force_http3=false, cookie_jar=None, cookies=None, follow_redirects=None))] fn $name( _py: Python, url: String, @@ -98,8 +98,9 @@ fn impit(_py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> { force_http3: Option, cookie_jar: Option>, cookies: Option>, + follow_redirects: Option, ) -> Result { - let client = Client::new(_py, None, None, None, None, None, None, None, None, cookie_jar, cookies, None, None); + let client = Client::new(_py, None, None, None, None, None, None, follow_redirects, None, cookie_jar, cookies, None, None); client?.$name(_py, url, content, data, headers, timeout, force_http3) } From 7cb7af9b491cc97f43d924426f4193b7e1d8a058 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jind=C5=99ich=20B=C3=A4r?= Date: Sun, 9 Nov 2025 09:24:42 +0100 Subject: [PATCH 03/13] fix(py): add `max_redirects` option to the anonymous client API --- impit-python/src/lib.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/impit-python/src/lib.rs b/impit-python/src/lib.rs index d02ca268..b1323431 100644 --- a/impit-python/src/lib.rs +++ b/impit-python/src/lib.rs @@ -87,7 +87,7 @@ fn impit(_py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> { ($($name:ident),*) => { $( #[pyfunction] - #[pyo3(signature = (url, content=None, data=None, headers=None, timeout=None, force_http3=false, cookie_jar=None, cookies=None, follow_redirects=None))] + #[pyo3(signature = (url, content=None, data=None, headers=None, timeout=None, force_http3=false, cookie_jar=None, cookies=None, follow_redirects=None, max_redirects=None))] fn $name( _py: Python, url: String, @@ -99,8 +99,9 @@ fn impit(_py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> { cookie_jar: Option>, cookies: Option>, follow_redirects: Option, + max_redirects: Option, ) -> Result { - let client = Client::new(_py, None, None, None, None, None, None, follow_redirects, None, cookie_jar, cookies, None, None); + let client = Client::new(_py, None, None, None, None, None, None, follow_redirects, max_redirects, cookie_jar, cookies, None, None); client?.$name(_py, url, content, data, headers, timeout, force_http3) } From 080f4ecebca586716d073674ca507764e5bd195c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jind=C5=99ich=20B=C3=A4r?= Date: Sun, 9 Nov 2025 09:24:52 +0100 Subject: [PATCH 04/13] chore: remove unused tests --- impit-python/test/no_client_test.py | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/impit-python/test/no_client_test.py b/impit-python/test/no_client_test.py index ca446db6..aa525b4a 100644 --- a/impit-python/test/no_client_test.py +++ b/impit-python/test/no_client_test.py @@ -137,7 +137,7 @@ def test_cookie_jar_works(self) -> None: assert len(cookies.jar) == 2 - def test_cookies_param_works(self, browser: Browser) -> None: + def test_cookies_param_works(self) -> None: cookies = Cookies({'preset-cookie': '123'}) response = impit.get( @@ -149,10 +149,12 @@ def test_cookies_param_works(self, browser: Browser) -> None: impit.get( get_httpbin_url('/cookies/set', query={'set-by-server': '321'}), + cookies=cookies, ) response = impit.get( get_httpbin_url('/cookies/'), + cookies=cookies, ).json() assert response['cookies'] == { @@ -218,20 +220,6 @@ def test_thread_server(self) -> None: assert response.status_code == 200 thread.join() - @pytest.mark.parametrize('addresses', [['127.0.0.1', '::ffff:127.0.0.1'], ['::1', '::1']]) - def test_local_address(self, browser: Browser, addresses: tuple[str, str]) -> None: - port_holder = [0] - thread = threading.Thread(target=thread_server, args=(port_holder,)) - thread.start() - time.sleep(0.1) - - [local_address, remote_address] = addresses - - response = impit.get(f'http://localhost:{port_holder[0]}/', timeout=5, local_address=local_address) - assert response.text == remote_address - assert response.status_code == 200 - thread.join() - class TestRequestBody: def test_passing_string_body(self) -> None: From f23d941f171b6e6046af551bdca4972761c01a18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jind=C5=99ich=20B=C3=A4r?= Date: Sun, 9 Nov 2025 09:33:24 +0100 Subject: [PATCH 05/13] fix(py): add `stream` method to the anonymous client API --- impit-python/python/impit/__init__.py | 2 ++ impit-python/python/impit/impit.pyi | 24 ++++++++++++++ impit-python/src/lib.rs | 46 +++++++++++++++++++++++++++ 3 files changed, 72 insertions(+) diff --git a/impit-python/python/impit/__init__.py b/impit-python/python/impit/__init__.py index 5607865a..26f40646 100644 --- a/impit-python/python/impit/__init__.py +++ b/impit-python/python/impit/__init__.py @@ -42,6 +42,7 @@ post, put, trace, + stream, ) __version__ = metadata.version('impit') @@ -88,6 +89,7 @@ 'post', 'put', 'trace', + 'stream', ] diff --git a/impit-python/python/impit/impit.pyi b/impit-python/python/impit/impit.pyi index 34b4f3c5..032fd047 100644 --- a/impit-python/python/impit/impit.pyi +++ b/impit-python/python/impit/impit.pyi @@ -1028,6 +1028,30 @@ class AsyncClient: """ +def stream( + method: str, + url: str, + content: bytes | bytearray | list[int] | None = None, + data: dict[str, str] | None = None, + headers: dict[str, str] | None = None, + timeout: float | None = None, + force_http3: bool | None = None, +) -> Response: + """Make a streaming request without creating a client instance. + + Args: + method: HTTP method (e.g., "get", "post") + url: URL to request + content: Raw content to send + data: Form data to send in request body + headers: HTTP headers + timeout: Request timeout in seconds + force_http3: Force HTTP/3 protocol + + Returns: + Response object + """ + def get( url: str, content: bytes | bytearray | list[int] | None = None, diff --git a/impit-python/src/lib.rs b/impit-python/src/lib.rs index b1323431..b9aea58f 100644 --- a/impit-python/src/lib.rs +++ b/impit-python/src/lib.rs @@ -113,5 +113,51 @@ fn impit(_py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> { http_no_client!(get, post, put, head, patch, delete, options, trace); + #[pyfunction] + #[pyo3(signature = (method, url, content=None, data=None, headers=None, timeout=None, force_http3=false, cookie_jar=None, cookies=None, follow_redirects=None, max_redirects=None))] + fn stream<'python>( + _py: Python<'python>, + method: &str, + url: String, + content: Option>, + data: Option, + headers: Option>, + timeout: Option, + force_http3: Option, + cookie_jar: Option>, + cookies: Option>, + follow_redirects: Option, + max_redirects: Option, + ) -> Result, PyErr> { + let client = Client::new( + _py, + None, + None, + None, + None, + None, + None, + follow_redirects, + max_redirects, + cookie_jar, + cookies, + None, + None, + ); + + client?.stream( + _py, + method, + url, + content, + data, + headers, + timeout, + force_http3, + ) + } + + m.add_function(wrap_pyfunction!(stream, m)?)?; + Ok(()) } From 5ddcb9199ba212fd60b69404f10b3f50eded51cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jind=C5=99ich=20B=C3=A4r?= Date: Sun, 9 Nov 2025 09:37:03 +0100 Subject: [PATCH 06/13] docs: add pydoc comments to the new anonymous client method params --- impit-python/python/impit/impit.pyi | 72 +++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/impit-python/python/impit/impit.pyi b/impit-python/python/impit/impit.pyi index 032fd047..d2a7a7fc 100644 --- a/impit-python/python/impit/impit.pyi +++ b/impit-python/python/impit/impit.pyi @@ -1036,6 +1036,10 @@ def stream( headers: dict[str, str] | None = None, timeout: float | None = None, force_http3: bool | None = None, + follow_redirects=None, + max_redirects=None, + cookie_jar=None, + cookies=None ) -> Response: """Make a streaming request without creating a client instance. @@ -1047,6 +1051,10 @@ def stream( headers: HTTP headers timeout: Request timeout in seconds force_http3: Force HTTP/3 protocol + follow_redirects: Whether to follow redirects (default: False) + max_redirects: Maximum number of redirects to follow (default: 20) + cookie_jar: Cookie jar to store cookies in. + cookies: httpx-compatible cookies object. Returns: Response object @@ -1059,6 +1067,10 @@ def get( headers: dict[str, str] | None = None, timeout: float | None = None, force_http3: bool | None = None, + follow_redirects=None, + max_redirects=None, + cookie_jar=None, + cookies=None ) -> Response: """Make a GET request without creating a client instance. @@ -1069,6 +1081,10 @@ def get( headers: HTTP headers timeout: Request timeout in seconds force_http3: Force HTTP/3 protocol + follow_redirects: Whether to follow redirects (default: False) + max_redirects: Maximum number of redirects to follow (default: 20) + cookie_jar: Cookie jar to store cookies in. + cookies: httpx-compatible cookies object. Returns: Response object @@ -1082,6 +1098,10 @@ def post( headers: dict[str, str] | None = None, timeout: float | None = None, force_http3: bool | None = None, + follow_redirects=None, + max_redirects=None, + cookie_jar=None, + cookies=None ) -> Response: """Make a POST request without creating a client instance. @@ -1092,6 +1112,10 @@ def post( headers: HTTP headers timeout: Request timeout in seconds force_http3: Force HTTP/3 protocol + follow_redirects: Whether to follow redirects (default: False) + max_redirects: Maximum number of redirects to follow (default: 20) + cookie_jar: Cookie jar to store cookies in. + cookies: httpx-compatible cookies object. Returns: Response object @@ -1105,6 +1129,10 @@ def put( headers: dict[str, str] | None = None, timeout: float | None = None, force_http3: bool | None = None, + follow_redirects=None, + max_redirects=None, + cookie_jar=None, + cookies=None ) -> Response: """Make a PUT request without creating a client instance. @@ -1115,6 +1143,10 @@ def put( headers: HTTP headers timeout: Request timeout in seconds force_http3: Force HTTP/3 protocol + follow_redirects: Whether to follow redirects (default: False) + max_redirects: Maximum number of redirects to follow (default: 20) + cookie_jar: Cookie jar to store cookies in. + cookies: httpx-compatible cookies object. Returns: Response object @@ -1128,6 +1160,10 @@ def patch( headers: dict[str, str] | None = None, timeout: float | None = None, force_http3: bool | None = None, + follow_redirects=None, + max_redirects=None, + cookie_jar=None, + cookies=None ) -> Response: """Make a PATCH request without creating a client instance. @@ -1138,6 +1174,10 @@ def patch( headers: HTTP headers timeout: Request timeout in seconds force_http3: Force HTTP/3 protocol + follow_redirects: Whether to follow redirects (default: False) + max_redirects: Maximum number of redirects to follow (default: 20) + cookie_jar: Cookie jar to store cookies in. + cookies: httpx-compatible cookies object. Returns: Response object @@ -1151,6 +1191,10 @@ def delete( headers: dict[str, str] | None = None, timeout: float | None = None, force_http3: bool | None = None, + follow_redirects=None, + max_redirects=None, + cookie_jar=None, + cookies=None ) -> Response: """Make a DELETE request without creating a client instance. @@ -1161,6 +1205,10 @@ def delete( headers: HTTP headers timeout: Request timeout in seconds force_http3: Force HTTP/3 protocol + follow_redirects: Whether to follow redirects (default: False) + max_redirects: Maximum number of redirects to follow (default: 20) + cookie_jar: Cookie jar to store cookies in. + cookies: httpx-compatible cookies object. Returns: Response object @@ -1174,6 +1222,10 @@ def head( headers: dict[str, str] | None = None, timeout: float | None = None, force_http3: bool | None = None, + follow_redirects=None, + max_redirects=None, + cookie_jar=None, + cookies=None ) -> Response: """Make a HEAD request without creating a client instance. @@ -1184,6 +1236,10 @@ def head( headers: HTTP headers timeout: Request timeout in seconds force_http3: Force HTTP/3 protocol + follow_redirects: Whether to follow redirects (default: False) + max_redirects: Maximum number of redirects to follow (default: 20) + cookie_jar: Cookie jar to store cookies in. + cookies: httpx-compatible cookies object. Returns: Response object @@ -1197,6 +1253,10 @@ def options( headers: dict[str, str] | None = None, timeout: float | None = None, force_http3: bool | None = None, + follow_redirects=None, + max_redirects=None, + cookie_jar=None, + cookies=None ) -> Response: """Make an OPTIONS request without creating a client instance. @@ -1207,6 +1267,10 @@ def options( headers: HTTP headers timeout: Request timeout in seconds (overrides default timeout) force_http3: Force HTTP/3 protocol + follow_redirects: Whether to follow redirects (default: False) + max_redirects: Maximum number of redirects to follow (default: 20) + cookie_jar: Cookie jar to store cookies in. + cookies: httpx-compatible cookies object. """ @@ -1217,6 +1281,10 @@ def trace( headers: dict[str, str] | None = None, timeout: float | None = None, force_http3: bool | None = None, + follow_redirects=None, + max_redirects=None, + cookie_jar=None, + cookies=None ) -> Response: """Make a TRACE request without creating a client instance. @@ -1227,4 +1295,8 @@ def trace( headers: HTTP headers timeout: Request timeout in seconds (overrides default timeout) force_http3: Force HTTP/3 protocol + follow_redirects: Whether to follow redirects (default: False) + max_redirects: Maximum number of redirects to follow (default: 20) + cookie_jar: Cookie jar to store cookies in. + cookies: httpx-compatible cookies object. """ From 2e3100853e07faa602de2c4b6d5f4a2ea4d823b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jind=C5=99ich=20B=C3=A4r?= Date: Sun, 9 Nov 2025 09:39:16 +0100 Subject: [PATCH 07/13] chore: run formatter --- impit-python/python/impit/__init__.py | 4 ++-- impit-python/test/no_client_test.py | 10 ++++++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/impit-python/python/impit/__init__.py b/impit-python/python/impit/__init__.py index 26f40646..d809788e 100644 --- a/impit-python/python/impit/__init__.py +++ b/impit-python/python/impit/__init__.py @@ -41,8 +41,8 @@ patch, post, put, - trace, stream, + trace, ) __version__ = metadata.version('impit') @@ -88,8 +88,8 @@ 'patch', 'post', 'put', - 'trace', 'stream', + 'trace', ] diff --git a/impit-python/test/no_client_test.py b/impit-python/test/no_client_test.py index aa525b4a..0177d1f4 100644 --- a/impit-python/test/no_client_test.py +++ b/impit-python/test/no_client_test.py @@ -6,8 +6,8 @@ import pytest -from impit import Browser, Client, Cookies, StreamClosed, StreamConsumed, TooManyRedirects import impit +from impit import Cookies, StreamClosed, StreamConsumed, TooManyRedirects from .httpbin import get_httpbin_url @@ -28,6 +28,7 @@ def thread_server(port_holder: list[int]) -> None: conn.close() server.close() + class TestBasicRequests: @pytest.mark.parametrize( ('protocol'), @@ -54,7 +55,11 @@ def test_headers_work(self) -> None: def test_cookies_nonstandard(self) -> None: cookies_jar = CookieJar() - impit.get(get_httpbin_url('/cookies/set', query={'set-by-server': '321'}), cookie_jar=cookies_jar, follow_redirects=True) + impit.get( + get_httpbin_url('/cookies/set', query={'set-by-server': '321'}), + cookie_jar=cookies_jar, + follow_redirects=True, + ) for cookie in cookies_jar: assert cookie.has_nonstandard_attr('HttpOnly') is not None @@ -303,6 +308,7 @@ def test_json(self) -> None: assert response.status_code == 200 assert response.json() == json.loads(response.text) + class TestStreamRequest: def test_read(self) -> None: with impit.stream('GET', get_httpbin_url('/')) as response: From 8ceb404ab47ec3cde6157c200fd4928f9e73321f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jind=C5=99ich=20B=C3=A4r?= Date: Sun, 9 Nov 2025 09:41:50 +0100 Subject: [PATCH 08/13] chore: fix `mypy` errorrs --- impit-python/python/impit/impit.pyi | 74 ++++++++++++++--------------- 1 file changed, 37 insertions(+), 37 deletions(-) diff --git a/impit-python/python/impit/impit.pyi b/impit-python/python/impit/impit.pyi index d2a7a7fc..b13b1ac8 100644 --- a/impit-python/python/impit/impit.pyi +++ b/impit-python/python/impit/impit.pyi @@ -1036,11 +1036,11 @@ def stream( headers: dict[str, str] | None = None, timeout: float | None = None, force_http3: bool | None = None, - follow_redirects=None, - max_redirects=None, - cookie_jar=None, - cookies=None -) -> Response: + follow_redirects: bool | None = None, + max_redirects: int | None = None, + cookie_jar: CookieJar | None = None, + cookies: Cookies | None = None, +) -> AbstractContextManager[Response]: """Make a streaming request without creating a client instance. Args: @@ -1067,10 +1067,10 @@ def get( headers: dict[str, str] | None = None, timeout: float | None = None, force_http3: bool | None = None, - follow_redirects=None, - max_redirects=None, - cookie_jar=None, - cookies=None + follow_redirects: bool | None = None, + max_redirects: int | None = None, + cookie_jar: CookieJar | None = None, + cookies: Cookies | None = None, ) -> Response: """Make a GET request without creating a client instance. @@ -1098,10 +1098,10 @@ def post( headers: dict[str, str] | None = None, timeout: float | None = None, force_http3: bool | None = None, - follow_redirects=None, - max_redirects=None, - cookie_jar=None, - cookies=None + follow_redirects: bool | None = None, + max_redirects: int | None = None, + cookie_jar: CookieJar | None = None, + cookies: Cookies | None = None, ) -> Response: """Make a POST request without creating a client instance. @@ -1129,10 +1129,10 @@ def put( headers: dict[str, str] | None = None, timeout: float | None = None, force_http3: bool | None = None, - follow_redirects=None, - max_redirects=None, - cookie_jar=None, - cookies=None + follow_redirects: bool | None = None, + max_redirects: int | None = None, + cookie_jar: CookieJar | None = None, + cookies: Cookies | None = None, ) -> Response: """Make a PUT request without creating a client instance. @@ -1160,10 +1160,10 @@ def patch( headers: dict[str, str] | None = None, timeout: float | None = None, force_http3: bool | None = None, - follow_redirects=None, - max_redirects=None, - cookie_jar=None, - cookies=None + follow_redirects: bool | None = None, + max_redirects: int | None = None, + cookie_jar: CookieJar | None = None, + cookies: Cookies | None = None, ) -> Response: """Make a PATCH request without creating a client instance. @@ -1191,10 +1191,10 @@ def delete( headers: dict[str, str] | None = None, timeout: float | None = None, force_http3: bool | None = None, - follow_redirects=None, - max_redirects=None, - cookie_jar=None, - cookies=None + follow_redirects: bool | None = None, + max_redirects: int | None = None, + cookie_jar: CookieJar | None = None, + cookies: Cookies | None = None, ) -> Response: """Make a DELETE request without creating a client instance. @@ -1222,10 +1222,10 @@ def head( headers: dict[str, str] | None = None, timeout: float | None = None, force_http3: bool | None = None, - follow_redirects=None, - max_redirects=None, - cookie_jar=None, - cookies=None + follow_redirects: bool | None = None, + max_redirects: int | None = None, + cookie_jar: CookieJar | None = None, + cookies: Cookies | None = None, ) -> Response: """Make a HEAD request without creating a client instance. @@ -1253,10 +1253,10 @@ def options( headers: dict[str, str] | None = None, timeout: float | None = None, force_http3: bool | None = None, - follow_redirects=None, - max_redirects=None, - cookie_jar=None, - cookies=None + follow_redirects: bool | None = None, + max_redirects: int | None = None, + cookie_jar: CookieJar | None = None, + cookies: Cookies | None = None, ) -> Response: """Make an OPTIONS request without creating a client instance. @@ -1281,10 +1281,10 @@ def trace( headers: dict[str, str] | None = None, timeout: float | None = None, force_http3: bool | None = None, - follow_redirects=None, - max_redirects=None, - cookie_jar=None, - cookies=None + follow_redirects: bool | None = None, + max_redirects: int | None = None, + cookie_jar: CookieJar | None = None, + cookies: Cookies | None = None, ) -> Response: """Make a TRACE request without creating a client instance. From ea9c6c5a4a565f159f4844da12f5285e070b64d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jind=C5=99ich=20B=C3=A4r?= Date: Sun, 9 Nov 2025 09:45:11 +0100 Subject: [PATCH 09/13] chore: fix `cookie_jar` test (pass `cookies.jar` each time) --- impit-python/test/no_client_test.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/impit-python/test/no_client_test.py b/impit-python/test/no_client_test.py index 0177d1f4..c6ce3f3a 100644 --- a/impit-python/test/no_client_test.py +++ b/impit-python/test/no_client_test.py @@ -129,10 +129,12 @@ def test_cookie_jar_works(self) -> None: impit.get( get_httpbin_url('/cookies/set', query={'set-by-server': '321'}), + cookie_jar=cookies.jar, ) response = impit.get( get_httpbin_url('/cookies/'), + cookie_jar=cookies.jar, ).json() assert response['cookies'] == { From 048335aed104c75db4a2652545e91568ecd6cb00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jind=C5=99ich=20B=C3=A4r?= Date: Mon, 10 Nov 2025 08:54:25 +0100 Subject: [PATCH 10/13] chore: add `http` proxy test to `impit-node` tests --- impit-node/package.json | 13 +++--- impit-node/test/basics.test.ts | 23 ++++++++-- impit-node/test/mock.server.ts | 12 ++++++ impit-node/yarn.lock | 78 ++++++++++++++++++++-------------- 4 files changed, 84 insertions(+), 42 deletions(-) diff --git a/impit-node/package.json b/impit-node/package.json index 609be2fc..5cd507dd 100644 --- a/impit-node/package.json +++ b/impit-node/package.json @@ -27,6 +27,7 @@ "@types/express": "^5.0.0", "@types/node": "^24.0.0", "express": "^5.0.0", + "proxy-chain": "^2.5.9", "socks-server-lib": "^0.0.3", "tough-cookie": "^6.0.0", "typedoc": "^0.28.13", @@ -53,13 +54,13 @@ "packageManager": "yarn@4.10.3", "description": "Impit for JavaScript", "optionalDependencies": { - "impit-darwin-x64": "0.7.0", "impit-darwin-arm64": "0.7.0", - "impit-win32-x64-msvc": "0.7.0", - "impit-win32-arm64-msvc": "0.7.0", + "impit-darwin-x64": "0.7.0", + "impit-linux-arm64-gnu": "0.7.0", + "impit-linux-arm64-musl": "0.7.0", "impit-linux-x64-gnu": "0.7.0", "impit-linux-x64-musl": "0.7.0", - "impit-linux-arm64-gnu": "0.7.0", - "impit-linux-arm64-musl": "0.7.0" + "impit-win32-arm64-msvc": "0.7.0", + "impit-win32-x64-msvc": "0.7.0" } -} \ No newline at end of file +} diff --git a/impit-node/test/basics.test.ts b/impit-node/test/basics.test.ts index 250f1726..30b9a9fd 100644 --- a/impit-node/test/basics.test.ts +++ b/impit-node/test/basics.test.ts @@ -2,10 +2,11 @@ import { test, describe, expect, beforeAll, afterAll } from 'vitest'; import { HttpMethod, Impit, Browser } from '../index.wrapper.js'; import type { Server } from 'net'; -import { routes, runServer } from './mock.server.js'; +import { routes, runProxyServer, runServer } from './mock.server.js'; import { CookieJar } from 'tough-cookie'; import { runSocksServer } from 'socks-server-lib'; +import { Server as ProxyServer } from 'proxy-chain'; function getHttpBinUrl(path: string, https?: boolean): string { https ??= true; @@ -28,6 +29,12 @@ async function getServer() { return localServer; } +let proxyServer: ProxyServer | null = null; +async function getProxyServer() { + proxyServer ??= await runProxyServer(3002); + return proxyServer; +} + let socksServer: Server | null = null; let socksConnectionCount = 0; beforeAll(async () => { @@ -36,6 +43,8 @@ beforeAll(async () => { await fetch(getHttpBinUrl('/get')); // Start the local server await getServer(); + // Start the proxy server + await getProxyServer() socksServer = await runSocksServer({ host: 'localhost', port: 7625, onData: () => { socksConnectionCount++; }}); }, 30e3); @@ -46,6 +55,10 @@ afterAll(async () => { const server = await getServer(); server?.close(() => res()) }), + new Promise(async (res) => { + const server = await getProxyServer(); + server?.close(true, () => res()) + }), Promise.race([ new Promise(res => { socksServer?.on('close', () => res()); @@ -130,10 +143,14 @@ describe.each([ ]); }); - test.each([['socks4'], ['socks5']])('supports %s proxy', async (proxyType) => { + test.each([ + { scheme: 'socks4', url: 'socks4://localhost:7625' }, + { scheme: 'socks5', url: 'socks5://localhost:7625' }, + { scheme: 'http', url: 'http://localhost:3002' }, + ])('supports %s proxy', async ({ scheme, url }) => { const impit = new Impit({ browser, - proxyUrl: `${proxyType}://localhost:7625`, + proxyUrl: url, }); const response = await impit.fetch( diff --git a/impit-node/test/mock.server.ts b/impit-node/test/mock.server.ts index b10aaf9b..5b1e9a42 100644 --- a/impit-node/test/mock.server.ts +++ b/impit-node/test/mock.server.ts @@ -1,5 +1,6 @@ import express from 'express'; import { Server } from 'http'; +import { Server as ProxyServer } from 'proxy-chain'; export const routes = { charset: { @@ -35,3 +36,14 @@ export async function runServer(port: number): Promise { }); }); } + +export async function runProxyServer(port: number): Promise { + const server = new ProxyServer({port}); + return new Promise((res, rej) => { + server.listen(() => { + res(server); + }).catch((err) => { + rej(err); + }); + }); +} diff --git a/impit-node/yarn.lock b/impit-node/yarn.lock index a9db7eff..808b121c 100644 --- a/impit-node/yarn.lock +++ b/impit-node/yarn.lock @@ -2500,58 +2500,58 @@ __metadata: languageName: node linkType: hard -"impit-darwin-arm64@npm:0.6.1": - version: 0.6.1 - resolution: "impit-darwin-arm64@npm:0.6.1" +"impit-darwin-arm64@npm:0.7.0": + version: 0.7.0 + resolution: "impit-darwin-arm64@npm:0.7.0" conditions: os=darwin & cpu=arm64 languageName: node linkType: hard -"impit-darwin-x64@npm:0.6.1": - version: 0.6.1 - resolution: "impit-darwin-x64@npm:0.6.1" +"impit-darwin-x64@npm:0.7.0": + version: 0.7.0 + resolution: "impit-darwin-x64@npm:0.7.0" conditions: os=darwin & cpu=x64 languageName: node linkType: hard -"impit-linux-arm64-gnu@npm:0.6.1": - version: 0.6.1 - resolution: "impit-linux-arm64-gnu@npm:0.6.1" +"impit-linux-arm64-gnu@npm:0.7.0": + version: 0.7.0 + resolution: "impit-linux-arm64-gnu@npm:0.7.0" conditions: os=linux & cpu=arm64 & libc=glibc languageName: node linkType: hard -"impit-linux-arm64-musl@npm:0.6.1": - version: 0.6.1 - resolution: "impit-linux-arm64-musl@npm:0.6.1" +"impit-linux-arm64-musl@npm:0.7.0": + version: 0.7.0 + resolution: "impit-linux-arm64-musl@npm:0.7.0" conditions: os=linux & cpu=arm64 & libc=musl languageName: node linkType: hard -"impit-linux-x64-gnu@npm:0.6.1": - version: 0.6.1 - resolution: "impit-linux-x64-gnu@npm:0.6.1" +"impit-linux-x64-gnu@npm:0.7.0": + version: 0.7.0 + resolution: "impit-linux-x64-gnu@npm:0.7.0" conditions: os=linux & cpu=x64 & libc=glibc languageName: node linkType: hard -"impit-linux-x64-musl@npm:0.6.1": - version: 0.6.1 - resolution: "impit-linux-x64-musl@npm:0.6.1" +"impit-linux-x64-musl@npm:0.7.0": + version: 0.7.0 + resolution: "impit-linux-x64-musl@npm:0.7.0" conditions: os=linux & cpu=x64 & libc=musl languageName: node linkType: hard -"impit-win32-arm64-msvc@npm:0.6.1": - version: 0.6.1 - resolution: "impit-win32-arm64-msvc@npm:0.6.1" +"impit-win32-arm64-msvc@npm:0.7.0": + version: 0.7.0 + resolution: "impit-win32-arm64-msvc@npm:0.7.0" conditions: os=win32 & cpu=arm64 languageName: node linkType: hard -"impit-win32-x64-msvc@npm:0.6.1": - version: 0.6.1 - resolution: "impit-win32-x64-msvc@npm:0.6.1" +"impit-win32-x64-msvc@npm:0.7.0": + version: 0.7.0 + resolution: "impit-win32-x64-msvc@npm:0.7.0" conditions: os=win32 & cpu=x64 languageName: node linkType: hard @@ -2564,14 +2564,15 @@ __metadata: "@types/express": "npm:^5.0.0" "@types/node": "npm:^24.0.0" express: "npm:^5.0.0" - impit-darwin-arm64: "npm:0.6.1" - impit-darwin-x64: "npm:0.6.1" - impit-linux-arm64-gnu: "npm:0.6.1" - impit-linux-arm64-musl: "npm:0.6.1" - impit-linux-x64-gnu: "npm:0.6.1" - impit-linux-x64-musl: "npm:0.6.1" - impit-win32-arm64-msvc: "npm:0.6.1" - impit-win32-x64-msvc: "npm:0.6.1" + impit-darwin-arm64: "npm:0.7.0" + impit-darwin-x64: "npm:0.7.0" + impit-linux-arm64-gnu: "npm:0.7.0" + impit-linux-arm64-musl: "npm:0.7.0" + impit-linux-x64-gnu: "npm:0.7.0" + impit-linux-x64-musl: "npm:0.7.0" + impit-win32-arm64-msvc: "npm:0.7.0" + impit-win32-x64-msvc: "npm:0.7.0" + proxy-chain: "npm:^2.5.9" socks-server-lib: "npm:^0.0.3" tough-cookie: "npm:^6.0.0" typedoc: "npm:^0.28.13" @@ -3098,6 +3099,17 @@ __metadata: languageName: node linkType: hard +"proxy-chain@npm:^2.5.9": + version: 2.5.9 + resolution: "proxy-chain@npm:2.5.9" + dependencies: + socks: "npm:^2.8.3" + socks-proxy-agent: "npm:^8.0.3" + tslib: "npm:^2.3.1" + checksum: 10c0/a55ebab793ec4a48b888536bb5ca4840768cb1550b966c82187580deb4c27fe7723a2609fb21f990006fd7a45a3d4c54eb38be4e9fe451027d1d9ce51628e73d + languageName: node + linkType: hard + "punycode.js@npm:^2.3.1": version: 2.3.1 resolution: "punycode.js@npm:2.3.1" @@ -3572,7 +3584,7 @@ __metadata: languageName: node linkType: hard -"tslib@npm:^2.4.0": +"tslib@npm:^2.3.1, tslib@npm:^2.4.0": version: 2.8.1 resolution: "tslib@npm:2.8.1" checksum: 10c0/9c4759110a19c53f992d9aae23aac5ced636e99887b51b9e61def52611732872ff7668757d4e4c61f19691e36f4da981cd9485e869b4a7408d689f6bf1f14e62 From 786a45d61f12e84dfd83d720a229ffccc9d8abce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jind=C5=99ich=20B=C3=A4r?= Date: Mon, 10 Nov 2025 10:16:16 +0100 Subject: [PATCH 11/13] chore: add basic `proxy` test for python clients --- impit-python/pyproject.toml | 1 + impit-python/test/async_client_test.py | 13 ++++++++ impit-python/test/basic_client_test.py | 12 ++++++++ impit-python/test/no_client_test.py | 11 ++++++- impit-python/test/setup_proxy.py | 42 ++++++++++++++++++++++++++ impit-python/uv.lock | 14 +++++++-- 6 files changed, 90 insertions(+), 3 deletions(-) create mode 100644 impit-python/test/setup_proxy.py diff --git a/impit-python/pyproject.toml b/impit-python/pyproject.toml index 78d17dc9..5f3c15e2 100644 --- a/impit-python/pyproject.toml +++ b/impit-python/pyproject.toml @@ -50,6 +50,7 @@ dev = [ "pytest", "ruff", "setuptools", + "pproxy>=2.7.9", ] docs = [ "sphinx>=7.4.7", diff --git a/impit-python/test/async_client_test.py b/impit-python/test/async_client_test.py index 60c9bfe6..8f4cc751 100644 --- a/impit-python/test/async_client_test.py +++ b/impit-python/test/async_client_test.py @@ -9,6 +9,7 @@ from impit import AsyncClient, Browser, Cookies, StreamClosed, StreamConsumed, TooManyRedirects from .httpbin import get_httpbin_url +from .setup_proxy import start_proxy_server def thread_server(port_holder: list[int]) -> None: @@ -261,6 +262,18 @@ async def test_methods_work(self, browser: Browser, method: str) -> None: await m(get_httpbin_url('/anything')) + @pytest.mark.asyncio + async def test_proxy(self, browser: Browser) -> None: + stop_proxy = start_proxy_server(3002) + impit = AsyncClient(browser=browser, proxy='http://127.0.0.1:3002') + target_url = 'https://crawlee.dev/' + + resp = await impit.get(target_url) + assert resp.status_code == 200 + assert 'Crawlee' in resp.text + + stop_proxy() + @pytest.mark.asyncio async def test_default_no_redirect(self, browser: Browser) -> None: impit = AsyncClient(browser=browser) diff --git a/impit-python/test/basic_client_test.py b/impit-python/test/basic_client_test.py index 297eb216..82cc300f 100644 --- a/impit-python/test/basic_client_test.py +++ b/impit-python/test/basic_client_test.py @@ -3,6 +3,7 @@ import threading import time from http.cookiejar import CookieJar +from .setup_proxy import start_proxy_server import pytest @@ -240,6 +241,17 @@ def test_methods_work(self, browser: Browser, method: str) -> None: m(get_httpbin_url('/anything')) + def test_proxy(self, browser: Browser) -> None: + stop_proxy = start_proxy_server(3002) + impit = Client(browser=browser, proxy='http://127.0.0.1:3002') + target_url = 'https://crawlee.dev/' + + resp = impit.get(target_url) + assert resp.status_code == 200 + assert 'Crawlee' in resp.text + + stop_proxy() + def test_default_no_redirect(self, browser: Browser) -> None: impit = Client(browser=browser) diff --git a/impit-python/test/no_client_test.py b/impit-python/test/no_client_test.py index c6ce3f3a..c1958faa 100644 --- a/impit-python/test/no_client_test.py +++ b/impit-python/test/no_client_test.py @@ -10,7 +10,7 @@ from impit import Cookies, StreamClosed, StreamConsumed, TooManyRedirects from .httpbin import get_httpbin_url - +from .setup_proxy import start_proxy_server def thread_server(port_holder: list[int]) -> None: server = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) @@ -188,6 +188,15 @@ def test_methods_work(self, method: str) -> None: m = getattr(impit, method.lower()) m(get_httpbin_url('/anything')) + def test_proxy(self) -> None: + stop_proxy = start_proxy_server(3002) + + resp = impit.get('https://crawlee.dev/', proxy='http://127.0.0.1:3002') + assert resp.status_code == 200 + assert 'Crawlee' in resp.text + + stop_proxy() + def test_default_no_redirect(self) -> None: target_url = 'https://crawlee.dev/' redirect_url = get_httpbin_url('/redirect-to', query={'url': target_url}) diff --git a/impit-python/test/setup_proxy.py b/impit-python/test/setup_proxy.py new file mode 100644 index 00000000..8a6a11ad --- /dev/null +++ b/impit-python/test/setup_proxy.py @@ -0,0 +1,42 @@ +import threading +import pproxy +import asyncio + +def start_proxy_server(port: int = 3002): + def run_proxy_server(stop_event): + server = pproxy.Server(f'http://0.0.0.0:{port}') + args = dict(rserver=[], + verbose=print) + + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + handler = loop.run_until_complete(server.start_server(args)) + + try: + while not stop_event.is_set(): + loop.run_until_complete(asyncio.sleep(0.1)) + except KeyboardInterrupt: + print('exit!') + + handler.close() + loop.run_until_complete(handler.wait_closed()) + loop.run_until_complete(loop.shutdown_asyncgens()) + loop.close() + + # Event to signal the proxy server to stop + stop_event = threading.Event() + + # Start the proxy server in a separate thread + proxy_thread = threading.Thread(target=run_proxy_server, args=(stop_event,), daemon=True) + proxy_thread.start() + + # wait a moment for the server to start + import time + time.sleep(1) + + # Return a function to stop the server + def stop_server(): + stop_event.set() + proxy_thread.join(1) + + return stop_server diff --git a/impit-python/uv.lock b/impit-python/uv.lock index 87fdf9e2..f3c88b49 100644 --- a/impit-python/uv.lock +++ b/impit-python/uv.lock @@ -258,7 +258,7 @@ name = "exceptiongroup" version = "1.3.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.13'" }, + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/0b/9f/a65090624ecf468cdca03533906e7c69ed7588582240cfe7cc9e770b50eb/exceptiongroup-1.3.0.tar.gz", hash = "sha256:b241f5885f560bc56a59ee63ca4c6a8bfa46ae4ad651af316d4e81817bb9fd88", size = 29749, upload-time = "2025-05-10T17:42:51.123Z" } wheels = [ @@ -311,12 +311,13 @@ wheels = [ [[package]] name = "impit" -version = "0.7.3" +version = "0.8.0" source = { editable = "." } [package.dev-dependencies] dev = [ { name = "mypy" }, + { name = "pproxy" }, { name = "pytest" }, { name = "pytest-asyncio" }, { name = "pytest-cov" }, @@ -339,6 +340,7 @@ docs = [ [package.metadata.requires-dev] dev = [ { name = "mypy" }, + { name = "pproxy", specifier = ">=2.7.9" }, { name = "pytest" }, { name = "pytest-asyncio" }, { name = "pytest-cov" }, @@ -566,6 +568,14 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, ] +[[package]] +name = "pproxy" +version = "2.7.9" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d4/c6/673a10a729061d2594b85aedd7dd2e470db4d54b12d4f95a306353bb2967/pproxy-2.7.9-py3-none-any.whl", hash = "sha256:a073d02616a47c43e1d20a547918c307dbda598c6d53869b165025f3cfe58e80", size = 42842, upload-time = "2024-01-16T11:33:35.286Z" }, +] + [[package]] name = "pygments" version = "2.19.2" From f144b14ad94511061d937dd8b979b89c467d219c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jind=C5=99ich=20B=C3=A4r?= Date: Mon, 10 Nov 2025 10:20:31 +0100 Subject: [PATCH 12/13] feat: add `proxy` option to anonymous client methods --- impit-python/python/impit/impit.pyi | 18 ++++++++++++++++++ impit-python/src/lib.rs | 10 ++++++---- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/impit-python/python/impit/impit.pyi b/impit-python/python/impit/impit.pyi index b13b1ac8..1e13afce 100644 --- a/impit-python/python/impit/impit.pyi +++ b/impit-python/python/impit/impit.pyi @@ -1040,6 +1040,7 @@ def stream( max_redirects: int | None = None, cookie_jar: CookieJar | None = None, cookies: Cookies | None = None, + proxy: str | None = None, ) -> AbstractContextManager[Response]: """Make a streaming request without creating a client instance. @@ -1055,6 +1056,7 @@ def stream( max_redirects: Maximum number of redirects to follow (default: 20) cookie_jar: Cookie jar to store cookies in. cookies: httpx-compatible cookies object. + proxy: Proxy URL to use to make the request. Returns: Response object @@ -1071,6 +1073,7 @@ def get( max_redirects: int | None = None, cookie_jar: CookieJar | None = None, cookies: Cookies | None = None, + proxy: str | None = None, ) -> Response: """Make a GET request without creating a client instance. @@ -1085,6 +1088,7 @@ def get( max_redirects: Maximum number of redirects to follow (default: 20) cookie_jar: Cookie jar to store cookies in. cookies: httpx-compatible cookies object. + proxy: Proxy URL to use to make the request. Returns: Response object @@ -1102,6 +1106,7 @@ def post( max_redirects: int | None = None, cookie_jar: CookieJar | None = None, cookies: Cookies | None = None, + proxy: str | None = None, ) -> Response: """Make a POST request without creating a client instance. @@ -1116,6 +1121,7 @@ def post( max_redirects: Maximum number of redirects to follow (default: 20) cookie_jar: Cookie jar to store cookies in. cookies: httpx-compatible cookies object. + proxy: Proxy URL to use to make the request. Returns: Response object @@ -1133,6 +1139,7 @@ def put( max_redirects: int | None = None, cookie_jar: CookieJar | None = None, cookies: Cookies | None = None, + proxy: str | None = None, ) -> Response: """Make a PUT request without creating a client instance. @@ -1147,6 +1154,7 @@ def put( max_redirects: Maximum number of redirects to follow (default: 20) cookie_jar: Cookie jar to store cookies in. cookies: httpx-compatible cookies object. + proxy: Proxy URL to use to make the request. Returns: Response object @@ -1164,6 +1172,7 @@ def patch( max_redirects: int | None = None, cookie_jar: CookieJar | None = None, cookies: Cookies | None = None, + proxy: str | None = None, ) -> Response: """Make a PATCH request without creating a client instance. @@ -1178,6 +1187,7 @@ def patch( max_redirects: Maximum number of redirects to follow (default: 20) cookie_jar: Cookie jar to store cookies in. cookies: httpx-compatible cookies object. + proxy: Proxy URL to use to make the request. Returns: Response object @@ -1195,6 +1205,7 @@ def delete( max_redirects: int | None = None, cookie_jar: CookieJar | None = None, cookies: Cookies | None = None, + proxy: str | None = None, ) -> Response: """Make a DELETE request without creating a client instance. @@ -1209,6 +1220,7 @@ def delete( max_redirects: Maximum number of redirects to follow (default: 20) cookie_jar: Cookie jar to store cookies in. cookies: httpx-compatible cookies object. + proxy: Proxy URL to use to make the request. Returns: Response object @@ -1226,6 +1238,7 @@ def head( max_redirects: int | None = None, cookie_jar: CookieJar | None = None, cookies: Cookies | None = None, + proxy: str | None = None, ) -> Response: """Make a HEAD request without creating a client instance. @@ -1240,6 +1253,7 @@ def head( max_redirects: Maximum number of redirects to follow (default: 20) cookie_jar: Cookie jar to store cookies in. cookies: httpx-compatible cookies object. + proxy: Proxy URL to use to make the request. Returns: Response object @@ -1257,6 +1271,7 @@ def options( max_redirects: int | None = None, cookie_jar: CookieJar | None = None, cookies: Cookies | None = None, + proxy: str | None = None, ) -> Response: """Make an OPTIONS request without creating a client instance. @@ -1271,6 +1286,7 @@ def options( max_redirects: Maximum number of redirects to follow (default: 20) cookie_jar: Cookie jar to store cookies in. cookies: httpx-compatible cookies object. + proxy: Proxy URL to use to make the request. """ @@ -1285,6 +1301,7 @@ def trace( max_redirects: int | None = None, cookie_jar: CookieJar | None = None, cookies: Cookies | None = None, + proxy: str | None = None, ) -> Response: """Make a TRACE request without creating a client instance. @@ -1299,4 +1316,5 @@ def trace( max_redirects: Maximum number of redirects to follow (default: 20) cookie_jar: Cookie jar to store cookies in. cookies: httpx-compatible cookies object. + proxy: Proxy URL to use to make the request. """ diff --git a/impit-python/src/lib.rs b/impit-python/src/lib.rs index b9aea58f..b6d2cf66 100644 --- a/impit-python/src/lib.rs +++ b/impit-python/src/lib.rs @@ -87,7 +87,7 @@ fn impit(_py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> { ($($name:ident),*) => { $( #[pyfunction] - #[pyo3(signature = (url, content=None, data=None, headers=None, timeout=None, force_http3=false, cookie_jar=None, cookies=None, follow_redirects=None, max_redirects=None))] + #[pyo3(signature = (url, content=None, data=None, headers=None, timeout=None, force_http3=false, cookie_jar=None, cookies=None, follow_redirects=None, max_redirects=None, proxy=None))] fn $name( _py: Python, url: String, @@ -100,8 +100,9 @@ fn impit(_py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> { cookies: Option>, follow_redirects: Option, max_redirects: Option, + proxy: Option, ) -> Result { - let client = Client::new(_py, None, None, None, None, None, None, follow_redirects, max_redirects, cookie_jar, cookies, None, None); + let client = Client::new(_py, None, None, proxy, None, None, None, follow_redirects, max_redirects, cookie_jar, cookies, None, None); client?.$name(_py, url, content, data, headers, timeout, force_http3) } @@ -114,7 +115,7 @@ fn impit(_py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> { http_no_client!(get, post, put, head, patch, delete, options, trace); #[pyfunction] - #[pyo3(signature = (method, url, content=None, data=None, headers=None, timeout=None, force_http3=false, cookie_jar=None, cookies=None, follow_redirects=None, max_redirects=None))] + #[pyo3(signature = (method, url, content=None, data=None, headers=None, timeout=None, force_http3=false, cookie_jar=None, cookies=None, follow_redirects=None, max_redirects=None, proxy=None))] fn stream<'python>( _py: Python<'python>, method: &str, @@ -128,12 +129,13 @@ fn impit(_py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> { cookies: Option>, follow_redirects: Option, max_redirects: Option, + proxy: Option, ) -> Result, PyErr> { let client = Client::new( _py, None, None, - None, + proxy, None, None, None, From b7e4c203f6e1c43fc6a24ccf8ab5cdaf4af2ba1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jind=C5=99ich=20B=C3=A4r?= Date: Mon, 10 Nov 2025 10:23:54 +0100 Subject: [PATCH 13/13] chore: run formatter --- impit-python/pyproject.toml | 4 ++++ impit-python/test/basic_client_test.py | 2 +- impit-python/test/no_client_test.py | 1 + impit-python/test/setup_proxy.py | 17 ++++++++++------- 4 files changed, 16 insertions(+), 8 deletions(-) diff --git a/impit-python/pyproject.toml b/impit-python/pyproject.toml index 5f3c15e2..6e1795cd 100644 --- a/impit-python/pyproject.toml +++ b/impit-python/pyproject.toml @@ -131,5 +131,9 @@ warn_return_any = true warn_unreachable = true warn_unused_ignores = true +[[tool.mypy.overrides]] +module = ["pproxy"] +ignore_missing_imports = true + [tool.uv.sources] stubdoc = { git = "https://github.com/bayashi-cl/stubdoc" } diff --git a/impit-python/test/basic_client_test.py b/impit-python/test/basic_client_test.py index 82cc300f..4b7cee04 100644 --- a/impit-python/test/basic_client_test.py +++ b/impit-python/test/basic_client_test.py @@ -3,13 +3,13 @@ import threading import time from http.cookiejar import CookieJar -from .setup_proxy import start_proxy_server import pytest from impit import Browser, Client, Cookies, StreamClosed, StreamConsumed, TooManyRedirects from .httpbin import get_httpbin_url +from .setup_proxy import start_proxy_server def thread_server(port_holder: list[int]) -> None: diff --git a/impit-python/test/no_client_test.py b/impit-python/test/no_client_test.py index c1958faa..6d871951 100644 --- a/impit-python/test/no_client_test.py +++ b/impit-python/test/no_client_test.py @@ -12,6 +12,7 @@ from .httpbin import get_httpbin_url from .setup_proxy import start_proxy_server + def thread_server(port_holder: list[int]) -> None: server = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) diff --git a/impit-python/test/setup_proxy.py b/impit-python/test/setup_proxy.py index 8a6a11ad..c2967b52 100644 --- a/impit-python/test/setup_proxy.py +++ b/impit-python/test/setup_proxy.py @@ -1,12 +1,15 @@ +import asyncio import threading +import time +import typing + import pproxy -import asyncio -def start_proxy_server(port: int = 3002): - def run_proxy_server(stop_event): + +def start_proxy_server(port: int = 3002) -> typing.Callable[[], None]: + def run_proxy_server(stop_event: threading.Event) -> None: server = pproxy.Server(f'http://0.0.0.0:{port}') - args = dict(rserver=[], - verbose=print) + args = dict(rserver=[], verbose=print) loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) @@ -31,11 +34,11 @@ def run_proxy_server(stop_event): proxy_thread.start() # wait a moment for the server to start - import time + time.sleep(1) # Return a function to stop the server - def stop_server(): + def stop_server() -> None: stop_event.set() proxy_thread.join(1)