Skip to content

Commit 80d9c95

Browse files
jhamonjhamon
authored
Add retry config for urllib3 requests (#491)
## Problem We want to automatically retry when errors occur ## Solution Implement `urllib3` retry configuration. We implemented the backup calculation with jitter ourselves because this is not available for all versions of `urllib3` that the SDK uses. ## Type of Change - [x] New feature (non-breaking change which adds functionality) ## Test Plan I created a mock server in `scripts/text-server.py` that simulates a high rate of failures (80% failure, only 1 in 5 requests succeed). `poetry run python3 scripts/test-server.py` Then I made some requests and observed logging to see what was going on. ```python >>> from pinecone import Pinecone >>> # Testing control plane with retries >>> pc = Pinecone(host='http://localhost:8000') >>> pc.list_indexes() >>> >>> # Data plane >>> idx = pc.Index(host='http://localhost:8000') >>> # enable debug logging >>> idx._vector_api.api_client.configuration.debug = True >>> >>> idx.upsert(vectors=[('1', [0.1, 0.2])]) DEBUG | pinecone.openapi_support.rest_urllib3:126 | Calling urllib3 request() DEBUG | urllib3.connectionpool:546 | http://localhost:8000 "POST /vectors/upsert HTTP/10" 500 None DEBUG | urllib3.util.retry:521 | Incremented Retry for (url='/vectors/upsert'): JitterRetry(total=4, connect=None, read=None, redirect=None, status=None) DEBUG | pinecone.openapi_support.retries:20 | Calculating retry backoff: 0.15197003184454544 (jitter: 0.15197003184454544) DEBUG | urllib3.connectionpool:943 | Retry: /vectors/upsert DEBUG | urllib3.connectionpool:546 | http://localhost:8000 "POST /vectors/upsert HTTP/10" 500 None DEBUG | urllib3.util.retry:521 | Incremented Retry for (url='/vectors/upsert'): JitterRetry(total=3, connect=None, read=None, redirect=None, status=None) DEBUG | pinecone.openapi_support.retries:20 | Calculating retry backoff: 0.7352149950424516 (jitter: 0.2352149950424516) DEBUG | urllib3.connectionpool:943 | Retry: /vectors/upsert DEBUG | urllib3.connectionpool:546 | http://localhost:8000 "POST /vectors/upsert HTTP/10" 500 None DEBUG | urllib3.util.retry:521 | Incremented Retry for (url='/vectors/upsert'): JitterRetry(total=2, connect=None, read=None, redirect=None, status=None) DEBUG | pinecone.openapi_support.retries:20 | Calculating retry backoff: 1.1307109027442626 (jitter: 0.13071090274426245) DEBUG | urllib3.connectionpool:943 | Retry: /vectors/upsert DEBUG | urllib3.connectionpool:546 | http://localhost:8000 "POST /vectors/upsert HTTP/10" 500 None DEBUG | urllib3.util.retry:521 | Incremented Retry for (url='/vectors/upsert'): JitterRetry(total=1, connect=None, read=None, redirect=None, status=None) DEBUG | pinecone.openapi_support.retries:20 | Calculating retry backoff: 2.142226695165083 (jitter: 0.14222669516508277) DEBUG | urllib3.connectionpool:943 | Retry: /vectors/upsert DEBUG | urllib3.connectionpool:546 | http://localhost:8000 "POST /vectors/upsert HTTP/10" 200 None DEBUG | pinecone.openapi_support.rest_urllib3:266 | response body: b'{"upsertedCount": 10}' DEBUG | pinecone.openapi_support.rest_utils:34 | response status: 200 {'upserted_count': 10} ```
1 parent c1688f6 commit 80d9c95

File tree

5 files changed

+215
-1
lines changed

5 files changed

+215
-1
lines changed

‎pinecone/openapi_support/rest_urllib3.py

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
from .rest_utils import raise_exceptions_or_return, RESTResponse, RestClientInterface
99

1010
import urllib3
11-
11+
from .retries import JitterRetry
1212
from .exceptions import PineconeApiException, PineconeApiValueError
1313

1414

@@ -52,6 +52,13 @@ def __init__(
5252

5353
if configuration.retries is not None:
5454
addition_pool_args["retries"] = configuration.retries
55+
else:
56+
addition_pool_args["retries"] = JitterRetry(
57+
total=5,
58+
backoff_factor=0.25,
59+
status_forcelist=(500, 502, 503, 504),
60+
allowed_methods=None,
61+
)
5562

5663
if configuration.socket_options is not None:
5764
addition_pool_args["socket_options"] = configuration.socket_options

‎pinecone/openapi_support/retries.py

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
import random
2+
from urllib3.util.retry import Retry
3+
import logging
4+
5+
logger = logging.getLogger(__name__)
6+
7+
8+
class JitterRetry(Retry):
9+
"""
10+
Retry with exponential back‑off with jitter.
11+
12+
The Retry class is being extended as built-in support for jitter was added only in urllib3 2.0.0.
13+
Jitter logic is following the official implementation with a constant jitter factor: https://github.com/urllib3/urllib3/blob/main/src/urllib3/util/retry.py
14+
"""
15+
16+
def get_backoff_time(self) -> float:
17+
backoff_value = super().get_backoff_time()
18+
jitter = random.random() * 0.25
19+
backoff_value += jitter
20+
logger.debug(f"Calculating retry backoff: {backoff_value} (jitter: {jitter})")
21+
return backoff_value

‎scripts/repl.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,7 @@ def cleanup_all(pc):
9898
"delete_all_collections": delete_all_collections,
9999
"delete_all_backups": delete_all_backups,
100100
"cleanup_all": cleanup_all,
101+
"pcl": Pinecone(host="http://localhost:8000"),
101102
# Add any other variables you want to have available in the REPL
102103
}
103104

‎scripts/test-server.py

Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
from http.server import BaseHTTPRequestHandler, HTTPServer
2+
import json
3+
4+
backups_response = {
5+
"data": [
6+
{
7+
"backup_id": "6f52240b-6397-481b-9767-748a2d4d3b65",
8+
"source_index_name": "jensparse",
9+
"source_index_id": "71ded150-2b8e-422d-9849-097f2c89d18b",
10+
"status": "Ready",
11+
"cloud": "aws",
12+
"region": "us-east-1",
13+
"tags": {},
14+
"name": "sparsebackup",
15+
"description": "",
16+
"dimension": 0,
17+
"record_count": 10000,
18+
"namespace_count": 1000,
19+
"size_bytes": 123456,
20+
"created_at": "2025-05-15T20:55:29.477794Z",
21+
}
22+
]
23+
}
24+
25+
indexes_response = {
26+
"indexes": [
27+
{
28+
"name": "jhamon-20250515-165135548-reorg-create-with-e",
29+
"metric": "dotproduct",
30+
"host": "jhamon-20250515-165135548-reorg-create-with-e-bt8x3su.svc.aped-4627-b74a.pinecone.io",
31+
"spec": {"serverless": {"cloud": "aws", "region": "us-east-1"}},
32+
"status": {"ready": True, "state": "Ready"},
33+
"vector_type": "sparse",
34+
"dimension": None,
35+
"deletion_protection": "disabled",
36+
"tags": {"env": "dev"},
37+
},
38+
{
39+
"name": "unexpected",
40+
"metric": "newmetric",
41+
"host": "jhamon-20250515-165135548-reorg-create-with-e-bt8x3su.svc.aped-4627-b74a.pinecone.io",
42+
"spec": {"serverless": {"cloud": "aws", "region": "us-east-1"}},
43+
"status": {"ready": False, "state": "UnknownStatus"},
44+
"vector_type": "sparse",
45+
"dimension": -1,
46+
"deletion_protection": "disabled",
47+
"tags": {"env": "dev"},
48+
},
49+
{
50+
"name": "wrong-types",
51+
"metric": 123,
52+
"host": "jhamon-20250515-165135548-reorg-create-with-e-bt8x3su.svc.aped-4627-b74a.pinecone.io",
53+
"spec": {"serverless": {"cloud": "aws", "region": "us-east-1"}},
54+
"status": {"ready": False, "state": "UnknownStatus"},
55+
"vector_type": None,
56+
"dimension": None,
57+
"deletion_protection": "asdf",
58+
"tags": None,
59+
},
60+
]
61+
}
62+
63+
index_description_response = {
64+
"name": "docs-example-dense",
65+
"vector_type": "dense",
66+
"metric": "cosine",
67+
"dimension": 1536,
68+
"status": {"ready": True, "state": "Ready"},
69+
"host": "docs-example-dense-govk0nt.svc.aped-4627-b74a.pinecone.io",
70+
"spec": {"serverless": {"region": "us-east-1", "cloud": "aws"}},
71+
"deletion_protection": "disabled",
72+
"tags": {"environment": "development"},
73+
}
74+
75+
upsert_response = {"upsertedCount": 10}
76+
77+
call_count = 0
78+
79+
80+
class MyHandler(BaseHTTPRequestHandler):
81+
def do_POST(self):
82+
global call_count
83+
call_count += 1
84+
85+
# Simulate a high rate of 500 errors
86+
if call_count % 5 != 0:
87+
self.send_response(500)
88+
self.end_headers()
89+
return
90+
91+
if self.path.startswith("/vectors/upsert"):
92+
self.send_response(200)
93+
self.send_header("Content-type", "application/json")
94+
self.end_headers()
95+
response = upsert_response
96+
self.wfile.write(json.dumps(response).encode())
97+
else:
98+
self.send_response(404)
99+
self.end_headers()
100+
101+
def do_GET(self):
102+
global call_count
103+
call_count += 1
104+
105+
# Simulate a high rate of 500 errors
106+
if call_count % 5 != 0:
107+
self.send_response(500)
108+
self.end_headers()
109+
return
110+
111+
if self.path.startswith("/backups"):
112+
self.send_response(200)
113+
self.send_header("Content-type", "application/json")
114+
self.end_headers()
115+
response = backups_response
116+
self.wfile.write(json.dumps(response).encode())
117+
elif self.path.startswith("/indexes/"):
118+
self.send_response(200)
119+
self.send_header("Content-type", "application/json")
120+
self.end_headers()
121+
response = index_description_response
122+
self.wfile.write(json.dumps(response).encode())
123+
elif self.path.startswith("/indexes"):
124+
self.send_response(200)
125+
self.send_header("Content-type", "application/json")
126+
self.end_headers()
127+
response = indexes_response
128+
self.wfile.write(json.dumps(response).encode())
129+
else:
130+
self.send_response(404)
131+
self.end_headers()
132+
133+
134+
server = HTTPServer(("localhost", 8000), MyHandler)
135+
print("Serving on http://localhost:8000")
136+
server.serve_forever()

‎tests/unit/openapi_support/test_retries.py

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
import pytest
2+
from unittest.mock import patch, MagicMock
3+
from urllib3.exceptions import MaxRetryError
4+
from urllib3.util.retry import Retry
5+
from pinecone.openapi_support.retries import JitterRetry
6+
7+
8+
def test_jitter_retry_backoff():
9+
"""Test that the backoff time includes jitter."""
10+
retry = JitterRetry(
11+
total=5,
12+
backoff_factor=0.25,
13+
backoff_max=3,
14+
status_forcelist=(500, 502, 503, 504),
15+
allowed_methods=None,
16+
)
17+
18+
# Mock the parent's get_backoff_time to return a fixed value
19+
with patch.object(Retry, "get_backoff_time", return_value=1.0):
20+
# Test multiple times to ensure jitter is added
21+
backoff_times = [retry.get_backoff_time() for _ in range(100)]
22+
23+
# All backoff times should be between 1.0 and 1.25
24+
assert all(1.0 <= t <= 1.25 for t in backoff_times)
25+
# Values should be different (jitter is working)
26+
assert len(set(backoff_times)) > 1
27+
28+
29+
def test_jitter_retry_behavior():
30+
"""Test that retries actually occur and respect the total count."""
31+
retry = JitterRetry(total=3)
32+
mock_response = MagicMock()
33+
mock_response.status = 500 # Simulate server error
34+
35+
# Simulate a failing request
36+
with pytest.raises(MaxRetryError) as exc_info:
37+
retry2 = retry.increment(
38+
method="GET", url="http://test.com", response=mock_response, error=None
39+
)
40+
retry3 = retry2.increment(
41+
method="GET", url="http://test.com", response=mock_response, error=None
42+
)
43+
retry4 = retry3.increment(
44+
method="GET", url="http://test.com", response=mock_response, error=None
45+
)
46+
retry4.increment(method="GET", url="http://test.com", response=mock_response, error=None)
47+
48+
# Verify the error contains the expected information
49+
assert "Max retries exceeded" in str(exc_info.value)

0 commit comments

Comments
 (0)