From e7a569795e76e6503e35dccde3a42a43edd785f9 Mon Sep 17 00:00:00 2001 From: Jen Hamon Date: Wed, 14 May 2025 07:08:50 -0400 Subject: [PATCH 1/7] WIP --- pinecone/openapi_support/rest_urllib3.py | 9 ++++++++- pinecone/openapi_support/retries.py | 16 ++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 pinecone/openapi_support/retries.py diff --git a/pinecone/openapi_support/rest_urllib3.py b/pinecone/openapi_support/rest_urllib3.py index 0c1a1c5a..a7def21e 100644 --- a/pinecone/openapi_support/rest_urllib3.py +++ b/pinecone/openapi_support/rest_urllib3.py @@ -8,7 +8,7 @@ from .rest_utils import raise_exceptions_or_return, RESTResponse, RestClientInterface import urllib3 - +from .retries import JitterRetry from .exceptions import PineconeApiException, PineconeApiValueError @@ -52,6 +52,13 @@ def __init__( if configuration.retries is not None: addition_pool_args["retries"] = configuration.retries + else: + addition_pool_args["retries"] = JitterRetry( + total=3, + backoff_factor=0.25, + status_forcelist=(500, 502, 503, 504), + allowed_methods=None, + ) if configuration.socket_options is not None: addition_pool_args["socket_options"] = configuration.socket_options diff --git a/pinecone/openapi_support/retries.py b/pinecone/openapi_support/retries.py new file mode 100644 index 00000000..8f631f4f --- /dev/null +++ b/pinecone/openapi_support/retries.py @@ -0,0 +1,16 @@ +import random +from urllib3.util.retry import Retry + + +class JitterRetry(Retry): + """ + Retry with exponential back‑off with jitter. + + The Retry class is being extended as built-in support for jitter was added only in urllib3 2.0.0. + Jitter logic is following the official implementation with a constant jitter factor: https://github.com/urllib3/urllib3/blob/main/src/urllib3/util/retry.py + """ + + def get_backoff_time(self) -> float: + backoff_value = super().get_backoff_time() + backoff_value += random.random() * 0.25 + return backoff_value From 8f7b8e1fe6fa95bc391dcef2014e1479334851f6 Mon Sep 17 00:00:00 2001 From: Jen Hamon Date: Wed, 14 May 2025 09:58:45 -0400 Subject: [PATCH 2/7] WIP --- tests/unit/openapi_support/test_retries.py | 54 +++++++++++ tests/unit/test_rest_urllib3.py | 105 +++++++++++++++++++++ 2 files changed, 159 insertions(+) create mode 100644 tests/unit/openapi_support/test_retries.py create mode 100644 tests/unit/test_rest_urllib3.py diff --git a/tests/unit/openapi_support/test_retries.py b/tests/unit/openapi_support/test_retries.py new file mode 100644 index 00000000..278f395b --- /dev/null +++ b/tests/unit/openapi_support/test_retries.py @@ -0,0 +1,54 @@ +import pytest +from unittest.mock import patch, MagicMock +from urllib3.exceptions import MaxRetryError +from pinecone.openapi_support.retries import JitterRetry + +def test_jitter_retry_backoff(): + """Test that the backoff time includes jitter.""" + retry = JitterRetry(total=3) + + # Mock the parent's get_backoff_time to return a fixed value + with patch.object(Retry, 'get_backoff_time', return_value=1.0): + # Test multiple times to ensure jitter is added + backoff_times = [retry.get_backoff_time() for _ in range(100)] + + # All backoff times should be between 1.0 and 1.25 + assert all(1.0 <= t <= 1.25 for t in backoff_times) + # Values should be different (jitter is working) + assert len(set(backoff_times)) > 1 + +def test_jitter_retry_behavior(): + """Test that retries actually occur and respect the total count.""" + retry = JitterRetry(total=3) + mock_response = MagicMock() + mock_response.status = 500 # Simulate server error + + # Simulate a failing request + with pytest.raises(MaxRetryError) as exc_info: + retry.increment( + method='GET', + url='http://test.com', + response=mock_response, + error=None + ) + + # Verify the error contains the expected information + assert "Max retries exceeded" in str(exc_info.value) + assert exc_info.value.reason.status == 500 + +def test_jitter_retry_success(): + """Test that retry stops on successful response.""" + retry = JitterRetry(total=3) + mock_response = MagicMock() + mock_response.status = 200 # Success response + + # Should not raise an exception for successful response + new_retry = retry.increment( + method='GET', + url='http://test.com', + response=mock_response, + error=None + ) + + # Verify retry count is decremented + assert new_retry.remaining == retry.total - 1 \ No newline at end of file diff --git a/tests/unit/test_rest_urllib3.py b/tests/unit/test_rest_urllib3.py new file mode 100644 index 00000000..256b5a95 --- /dev/null +++ b/tests/unit/test_rest_urllib3.py @@ -0,0 +1,105 @@ +import pytest +from unittest.mock import patch, MagicMock +import urllib3 +from urllib3.exceptions import MaxRetryError +from pinecone.openapi_support.rest_urllib3 import Urllib3RestClient +from pinecone.config.openapi_configuration import Configuration + +class TestUrllib3RestClient: + @pytest.fixture + def config(self): + return Configuration(api_key="test-key") + + @pytest.fixture + def client(self, config): + return Urllib3RestClient(config) + + def test_retry_on_500_error(self, client): + # Mock response that fails with 500 + mock_response = MagicMock() + mock_response.status = 500 + mock_response.data = b'{"error": "Internal Server Error"}' + mock_response.headers = {} + mock_response.reason = "Internal Server Error" + + # Mock pool manager to fail twice then succeed + with patch.object(client.pool_manager, 'request') as mock_request: + mock_request.side_effect = [ + urllib3.exceptions.HTTPError(response=mock_response), + urllib3.exceptions.HTTPError(response=mock_response), + mock_response # Success on third try + ] + + # Make request + response = client.request( + method="GET", + url="https://api.pinecone.io/test", + headers={"Authorization": "test-key"} + ) + + # Verify request was made 3 times (initial + 2 retries) + assert mock_request.call_count == 3 + + # Verify the response is successful + assert response.status == 200 + + def test_max_retries_exceeded(self, client): + # Mock response that always fails with 500 + mock_response = MagicMock() + mock_response.status = 500 + mock_response.data = b'{"error": "Internal Server Error"}' + mock_response.headers = {} + mock_response.reason = "Internal Server Error" + + # Mock pool manager to always fail + with patch.object(client.pool_manager, 'request') as mock_request: + mock_request.side_effect = urllib3.exceptions.HTTPError(response=mock_response) + + # Make request and expect MaxRetryError + with pytest.raises(MaxRetryError): + client.request( + method="GET", + url="https://api.pinecone.io/test", + headers={"Authorization": "test-key"} + ) + + # Verify request was made 4 times (initial + 3 retries) + assert mock_request.call_count == 4 + + def test_custom_retry_config(self): + # Create custom retry configuration + custom_retry = urllib3.Retry( + total=2, + backoff_factor=0.5, + status_forcelist=(500, 502, 503, 504) + ) + + config = Configuration(api_key="test-key", retries=custom_retry) + client = Urllib3RestClient(config) + + # Mock response that fails with 500 + mock_response = MagicMock() + mock_response.status = 500 + mock_response.data = b'{"error": "Internal Server Error"}' + mock_response.headers = {} + mock_response.reason = "Internal Server Error" + + # Mock pool manager to fail once then succeed + with patch.object(client.pool_manager, 'request') as mock_request: + mock_request.side_effect = [ + urllib3.exceptions.HTTPError(response=mock_response), + mock_response # Success on second try + ] + + # Make request + response = client.request( + method="GET", + url="https://api.pinecone.io/test", + headers={"Authorization": "test-key"} + ) + + # Verify request was made 2 times (initial + 1 retry) + assert mock_request.call_count == 2 + + # Verify the response is successful + assert response.status == 200 \ No newline at end of file From 59415126b69f2d65239d7a7155a614db37b25103 Mon Sep 17 00:00:00 2001 From: Jen Hamon Date: Wed, 14 May 2025 09:59:32 -0400 Subject: [PATCH 3/7] WIP --- tests/unit/openapi_support/test_retries.py | 31 +++++++++------------- tests/unit/test_rest_urllib3.py | 25 +++++++++-------- 2 files changed, 25 insertions(+), 31 deletions(-) diff --git a/tests/unit/openapi_support/test_retries.py b/tests/unit/openapi_support/test_retries.py index 278f395b..6ca927d7 100644 --- a/tests/unit/openapi_support/test_retries.py +++ b/tests/unit/openapi_support/test_retries.py @@ -3,52 +3,47 @@ from urllib3.exceptions import MaxRetryError from pinecone.openapi_support.retries import JitterRetry + def test_jitter_retry_backoff(): """Test that the backoff time includes jitter.""" retry = JitterRetry(total=3) - + # Mock the parent's get_backoff_time to return a fixed value - with patch.object(Retry, 'get_backoff_time', return_value=1.0): + with patch.object(Retry, "get_backoff_time", return_value=1.0): # Test multiple times to ensure jitter is added backoff_times = [retry.get_backoff_time() for _ in range(100)] - + # All backoff times should be between 1.0 and 1.25 assert all(1.0 <= t <= 1.25 for t in backoff_times) # Values should be different (jitter is working) assert len(set(backoff_times)) > 1 + def test_jitter_retry_behavior(): """Test that retries actually occur and respect the total count.""" retry = JitterRetry(total=3) mock_response = MagicMock() mock_response.status = 500 # Simulate server error - + # Simulate a failing request with pytest.raises(MaxRetryError) as exc_info: - retry.increment( - method='GET', - url='http://test.com', - response=mock_response, - error=None - ) - + retry.increment(method="GET", url="http://test.com", response=mock_response, error=None) + # Verify the error contains the expected information assert "Max retries exceeded" in str(exc_info.value) assert exc_info.value.reason.status == 500 + def test_jitter_retry_success(): """Test that retry stops on successful response.""" retry = JitterRetry(total=3) mock_response = MagicMock() mock_response.status = 200 # Success response - + # Should not raise an exception for successful response new_retry = retry.increment( - method='GET', - url='http://test.com', - response=mock_response, - error=None + method="GET", url="http://test.com", response=mock_response, error=None ) - + # Verify retry count is decremented - assert new_retry.remaining == retry.total - 1 \ No newline at end of file + assert new_retry.remaining == retry.total - 1 diff --git a/tests/unit/test_rest_urllib3.py b/tests/unit/test_rest_urllib3.py index 256b5a95..ac5a60d6 100644 --- a/tests/unit/test_rest_urllib3.py +++ b/tests/unit/test_rest_urllib3.py @@ -5,6 +5,7 @@ from pinecone.openapi_support.rest_urllib3 import Urllib3RestClient from pinecone.config.openapi_configuration import Configuration + class TestUrllib3RestClient: @pytest.fixture def config(self): @@ -23,18 +24,18 @@ def test_retry_on_500_error(self, client): mock_response.reason = "Internal Server Error" # Mock pool manager to fail twice then succeed - with patch.object(client.pool_manager, 'request') as mock_request: + with patch.object(client.pool_manager, "request") as mock_request: mock_request.side_effect = [ urllib3.exceptions.HTTPError(response=mock_response), urllib3.exceptions.HTTPError(response=mock_response), - mock_response # Success on third try + mock_response, # Success on third try ] # Make request response = client.request( method="GET", url="https://api.pinecone.io/test", - headers={"Authorization": "test-key"} + headers={"Authorization": "test-key"}, ) # Verify request was made 3 times (initial + 2 retries) @@ -52,7 +53,7 @@ def test_max_retries_exceeded(self, client): mock_response.reason = "Internal Server Error" # Mock pool manager to always fail - with patch.object(client.pool_manager, 'request') as mock_request: + with patch.object(client.pool_manager, "request") as mock_request: mock_request.side_effect = urllib3.exceptions.HTTPError(response=mock_response) # Make request and expect MaxRetryError @@ -60,7 +61,7 @@ def test_max_retries_exceeded(self, client): client.request( method="GET", url="https://api.pinecone.io/test", - headers={"Authorization": "test-key"} + headers={"Authorization": "test-key"}, ) # Verify request was made 4 times (initial + 3 retries) @@ -69,11 +70,9 @@ def test_max_retries_exceeded(self, client): def test_custom_retry_config(self): # Create custom retry configuration custom_retry = urllib3.Retry( - total=2, - backoff_factor=0.5, - status_forcelist=(500, 502, 503, 504) + total=2, backoff_factor=0.5, status_forcelist=(500, 502, 503, 504) ) - + config = Configuration(api_key="test-key", retries=custom_retry) client = Urllib3RestClient(config) @@ -85,21 +84,21 @@ def test_custom_retry_config(self): mock_response.reason = "Internal Server Error" # Mock pool manager to fail once then succeed - with patch.object(client.pool_manager, 'request') as mock_request: + with patch.object(client.pool_manager, "request") as mock_request: mock_request.side_effect = [ urllib3.exceptions.HTTPError(response=mock_response), - mock_response # Success on second try + mock_response, # Success on second try ] # Make request response = client.request( method="GET", url="https://api.pinecone.io/test", - headers={"Authorization": "test-key"} + headers={"Authorization": "test-key"}, ) # Verify request was made 2 times (initial + 1 retry) assert mock_request.call_count == 2 # Verify the response is successful - assert response.status == 200 \ No newline at end of file + assert response.status == 200 From 338db766d6dcdd1c3d217e0b5eb9dc2ef3a57e3c Mon Sep 17 00:00:00 2001 From: Jen Hamon Date: Fri, 16 May 2025 10:41:50 -0400 Subject: [PATCH 4/7] Add retries config for urllib3 --- pinecone/openapi_support/rest_urllib3.py | 3 +- pinecone/openapi_support/retries.py | 7 +- scripts/repl.py | 1 + scripts/test-server.py | 136 +++++++++++++++++++++++ 4 files changed, 145 insertions(+), 2 deletions(-) create mode 100644 scripts/test-server.py diff --git a/pinecone/openapi_support/rest_urllib3.py b/pinecone/openapi_support/rest_urllib3.py index a7def21e..e4e87423 100644 --- a/pinecone/openapi_support/rest_urllib3.py +++ b/pinecone/openapi_support/rest_urllib3.py @@ -54,8 +54,9 @@ def __init__( addition_pool_args["retries"] = configuration.retries else: addition_pool_args["retries"] = JitterRetry( - total=3, + total=5, backoff_factor=0.25, + backoff_max=3, status_forcelist=(500, 502, 503, 504), allowed_methods=None, ) diff --git a/pinecone/openapi_support/retries.py b/pinecone/openapi_support/retries.py index 8f631f4f..2b91a31d 100644 --- a/pinecone/openapi_support/retries.py +++ b/pinecone/openapi_support/retries.py @@ -1,5 +1,8 @@ import random from urllib3.util.retry import Retry +import logging + +logger = logging.getLogger(__name__) class JitterRetry(Retry): @@ -12,5 +15,7 @@ class JitterRetry(Retry): def get_backoff_time(self) -> float: backoff_value = super().get_backoff_time() - backoff_value += random.random() * 0.25 + jitter = random.random() * 0.25 + backoff_value += jitter + logger.debug(f"Calculating retry backoff: {backoff_value} (jitter: {jitter})") return backoff_value diff --git a/scripts/repl.py b/scripts/repl.py index 88c218b2..f9f85000 100644 --- a/scripts/repl.py +++ b/scripts/repl.py @@ -88,6 +88,7 @@ def cleanup_all(pc): "delete_all_collections": delete_all_collections, "delete_all_backups": delete_all_backups, "cleanup_all": cleanup_all, + "pcl": Pinecone(host="http://localhost:8000"), # Add any other variables you want to have available in the REPL } diff --git a/scripts/test-server.py b/scripts/test-server.py new file mode 100644 index 00000000..784d510d --- /dev/null +++ b/scripts/test-server.py @@ -0,0 +1,136 @@ +from http.server import BaseHTTPRequestHandler, HTTPServer +import json + +backups_response = { + "data": [ + { + "backup_id": "6f52240b-6397-481b-9767-748a2d4d3b65", + "source_index_name": "jensparse", + "source_index_id": "71ded150-2b8e-422d-9849-097f2c89d18b", + "status": "Ready", + "cloud": "aws", + "region": "us-east-1", + "tags": {}, + "name": "sparsebackup", + "description": "", + "dimension": 0, + "record_count": 10000, + "namespace_count": 1000, + "size_bytes": 123456, + "created_at": "2025-05-15T20:55:29.477794Z", + } + ] +} + +indexes_response = { + "indexes": [ + { + "name": "jhamon-20250515-165135548-reorg-create-with-e", + "metric": "dotproduct", + "host": "jhamon-20250515-165135548-reorg-create-with-e-bt8x3su.svc.aped-4627-b74a.pinecone.io", + "spec": {"serverless": {"cloud": "aws", "region": "us-east-1"}}, + "status": {"ready": True, "state": "Ready"}, + "vector_type": "sparse", + "dimension": None, + "deletion_protection": "disabled", + "tags": {"env": "dev"}, + }, + { + "name": "unexpected", + "metric": "newmetric", + "host": "jhamon-20250515-165135548-reorg-create-with-e-bt8x3su.svc.aped-4627-b74a.pinecone.io", + "spec": {"serverless": {"cloud": "aws", "region": "us-east-1"}}, + "status": {"ready": False, "state": "UnknownStatus"}, + "vector_type": "sparse", + "dimension": -1, + "deletion_protection": "disabled", + "tags": {"env": "dev"}, + }, + { + "name": "wrong-types", + "metric": 123, + "host": "jhamon-20250515-165135548-reorg-create-with-e-bt8x3su.svc.aped-4627-b74a.pinecone.io", + "spec": {"serverless": {"cloud": "aws", "region": "us-east-1"}}, + "status": {"ready": False, "state": "UnknownStatus"}, + "vector_type": None, + "dimension": None, + "deletion_protection": "asdf", + "tags": None, + }, + ] +} + +index_description_response = { + "name": "docs-example-dense", + "vector_type": "dense", + "metric": "cosine", + "dimension": 1536, + "status": {"ready": True, "state": "Ready"}, + "host": "docs-example-dense-govk0nt.svc.aped-4627-b74a.pinecone.io", + "spec": {"serverless": {"region": "us-east-1", "cloud": "aws"}}, + "deletion_protection": "disabled", + "tags": {"environment": "development"}, +} + +upsert_response = {"upsertedCount": 10} + +call_count = 0 + + +class MyHandler(BaseHTTPRequestHandler): + def do_POST(self): + global call_count + call_count += 1 + + # Simulate a high rate of 500 errors + if call_count % 5 != 0: + self.send_response(500) + self.end_headers() + return + + if self.path.startswith("/vectors/upsert"): + self.send_response(200) + self.send_header("Content-type", "application/json") + self.end_headers() + response = upsert_response + self.wfile.write(json.dumps(response).encode()) + else: + self.send_response(404) + self.end_headers() + + def do_GET(self): + global call_count + call_count += 1 + + # Simulate a high rate of 500 errors + if call_count % 5 != 0: + self.send_response(500) + self.end_headers() + return + + if self.path.startswith("/backups"): + self.send_response(200) + self.send_header("Content-type", "application/json") + self.end_headers() + response = backups_response + self.wfile.write(json.dumps(response).encode()) + elif self.path.startswith("/indexes/"): + self.send_response(200) + self.send_header("Content-type", "application/json") + self.end_headers() + response = index_description_response + self.wfile.write(json.dumps(response).encode()) + elif self.path.startswith("/indexes"): + self.send_response(200) + self.send_header("Content-type", "application/json") + self.end_headers() + response = indexes_response + self.wfile.write(json.dumps(response).encode()) + else: + self.send_response(404) + self.end_headers() + + +server = HTTPServer(("localhost", 8000), MyHandler) +print("Serving on http://localhost:8000") +server.serve_forever() From 916938dd12e3e1ace99a06e9c78b8e98f89ef2cf Mon Sep 17 00:00:00 2001 From: Jen Hamon Date: Fri, 16 May 2025 11:01:40 -0400 Subject: [PATCH 5/7] Update unit tests --- tests/unit/openapi_support/test_retries.py | 36 +++++++++++----------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/tests/unit/openapi_support/test_retries.py b/tests/unit/openapi_support/test_retries.py index 6ca927d7..5f31221d 100644 --- a/tests/unit/openapi_support/test_retries.py +++ b/tests/unit/openapi_support/test_retries.py @@ -1,12 +1,19 @@ import pytest from unittest.mock import patch, MagicMock from urllib3.exceptions import MaxRetryError +from urllib3.util.retry import Retry from pinecone.openapi_support.retries import JitterRetry def test_jitter_retry_backoff(): """Test that the backoff time includes jitter.""" - retry = JitterRetry(total=3) + retry = JitterRetry( + total=5, + backoff_factor=0.25, + backoff_max=3, + status_forcelist=(500, 502, 503, 504), + allowed_methods=None, + ) # Mock the parent's get_backoff_time to return a fixed value with patch.object(Retry, "get_backoff_time", return_value=1.0): @@ -27,23 +34,16 @@ def test_jitter_retry_behavior(): # Simulate a failing request with pytest.raises(MaxRetryError) as exc_info: - retry.increment(method="GET", url="http://test.com", response=mock_response, error=None) + retry2 = retry.increment( + method="GET", url="http://test.com", response=mock_response, error=None + ) + retry3 = retry2.increment( + method="GET", url="http://test.com", response=mock_response, error=None + ) + retry4 = retry3.increment( + method="GET", url="http://test.com", response=mock_response, error=None + ) + retry4.increment(method="GET", url="http://test.com", response=mock_response, error=None) # Verify the error contains the expected information assert "Max retries exceeded" in str(exc_info.value) - assert exc_info.value.reason.status == 500 - - -def test_jitter_retry_success(): - """Test that retry stops on successful response.""" - retry = JitterRetry(total=3) - mock_response = MagicMock() - mock_response.status = 200 # Success response - - # Should not raise an exception for successful response - new_retry = retry.increment( - method="GET", url="http://test.com", response=mock_response, error=None - ) - - # Verify retry count is decremented - assert new_retry.remaining == retry.total - 1 From bcc3ac81d8b205781af14b57b2019b2b4adb23f3 Mon Sep 17 00:00:00 2001 From: Jen Hamon Date: Fri, 16 May 2025 11:02:59 -0400 Subject: [PATCH 6/7] Fix mypy errors --- pinecone/openapi_support/rest_urllib3.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pinecone/openapi_support/rest_urllib3.py b/pinecone/openapi_support/rest_urllib3.py index e4e87423..f310ca99 100644 --- a/pinecone/openapi_support/rest_urllib3.py +++ b/pinecone/openapi_support/rest_urllib3.py @@ -56,7 +56,6 @@ def __init__( addition_pool_args["retries"] = JitterRetry( total=5, backoff_factor=0.25, - backoff_max=3, status_forcelist=(500, 502, 503, 504), allowed_methods=None, ) From 3826a255c8eec56050e9d1ac31d9ed2d8fc92e38 Mon Sep 17 00:00:00 2001 From: Jen Hamon Date: Fri, 16 May 2025 11:22:35 -0400 Subject: [PATCH 7/7] Remove extra test --- tests/unit/test_rest_urllib3.py | 104 -------------------------------- 1 file changed, 104 deletions(-) delete mode 100644 tests/unit/test_rest_urllib3.py diff --git a/tests/unit/test_rest_urllib3.py b/tests/unit/test_rest_urllib3.py deleted file mode 100644 index ac5a60d6..00000000 --- a/tests/unit/test_rest_urllib3.py +++ /dev/null @@ -1,104 +0,0 @@ -import pytest -from unittest.mock import patch, MagicMock -import urllib3 -from urllib3.exceptions import MaxRetryError -from pinecone.openapi_support.rest_urllib3 import Urllib3RestClient -from pinecone.config.openapi_configuration import Configuration - - -class TestUrllib3RestClient: - @pytest.fixture - def config(self): - return Configuration(api_key="test-key") - - @pytest.fixture - def client(self, config): - return Urllib3RestClient(config) - - def test_retry_on_500_error(self, client): - # Mock response that fails with 500 - mock_response = MagicMock() - mock_response.status = 500 - mock_response.data = b'{"error": "Internal Server Error"}' - mock_response.headers = {} - mock_response.reason = "Internal Server Error" - - # Mock pool manager to fail twice then succeed - with patch.object(client.pool_manager, "request") as mock_request: - mock_request.side_effect = [ - urllib3.exceptions.HTTPError(response=mock_response), - urllib3.exceptions.HTTPError(response=mock_response), - mock_response, # Success on third try - ] - - # Make request - response = client.request( - method="GET", - url="https://api.pinecone.io/test", - headers={"Authorization": "test-key"}, - ) - - # Verify request was made 3 times (initial + 2 retries) - assert mock_request.call_count == 3 - - # Verify the response is successful - assert response.status == 200 - - def test_max_retries_exceeded(self, client): - # Mock response that always fails with 500 - mock_response = MagicMock() - mock_response.status = 500 - mock_response.data = b'{"error": "Internal Server Error"}' - mock_response.headers = {} - mock_response.reason = "Internal Server Error" - - # Mock pool manager to always fail - with patch.object(client.pool_manager, "request") as mock_request: - mock_request.side_effect = urllib3.exceptions.HTTPError(response=mock_response) - - # Make request and expect MaxRetryError - with pytest.raises(MaxRetryError): - client.request( - method="GET", - url="https://api.pinecone.io/test", - headers={"Authorization": "test-key"}, - ) - - # Verify request was made 4 times (initial + 3 retries) - assert mock_request.call_count == 4 - - def test_custom_retry_config(self): - # Create custom retry configuration - custom_retry = urllib3.Retry( - total=2, backoff_factor=0.5, status_forcelist=(500, 502, 503, 504) - ) - - config = Configuration(api_key="test-key", retries=custom_retry) - client = Urllib3RestClient(config) - - # Mock response that fails with 500 - mock_response = MagicMock() - mock_response.status = 500 - mock_response.data = b'{"error": "Internal Server Error"}' - mock_response.headers = {} - mock_response.reason = "Internal Server Error" - - # Mock pool manager to fail once then succeed - with patch.object(client.pool_manager, "request") as mock_request: - mock_request.side_effect = [ - urllib3.exceptions.HTTPError(response=mock_response), - mock_response, # Success on second try - ] - - # Make request - response = client.request( - method="GET", - url="https://api.pinecone.io/test", - headers={"Authorization": "test-key"}, - ) - - # Verify request was made 2 times (initial + 1 retry) - assert mock_request.call_count == 2 - - # Verify the response is successful - assert response.status == 200