Skip to content

Add retry config for urllib3 requests #491

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 7 commits into from
May 16, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion pinecone/openapi_support/rest_urllib3.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
from .rest_utils import raise_exceptions_or_return, RESTResponse, RestClientInterface

import urllib3

from .retries import JitterRetry
from .exceptions import PineconeApiException, PineconeApiValueError


Expand Down Expand Up @@ -52,6 +52,13 @@ def __init__(

if configuration.retries is not None:
addition_pool_args["retries"] = configuration.retries
else:
addition_pool_args["retries"] = JitterRetry(
total=5,
backoff_factor=0.25,
status_forcelist=(500, 502, 503, 504),
allowed_methods=None,
)

if configuration.socket_options is not None:
addition_pool_args["socket_options"] = configuration.socket_options
Expand Down
21 changes: 21 additions & 0 deletions pinecone/openapi_support/retries.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
import random
from urllib3.util.retry import Retry
import logging

logger = logging.getLogger(__name__)


class JitterRetry(Retry):
"""
Retry with exponential back‑off with jitter.

The Retry class is being extended as built-in support for jitter was added only in urllib3 2.0.0.
Jitter logic is following the official implementation with a constant jitter factor: https://github.com/urllib3/urllib3/blob/main/src/urllib3/util/retry.py
"""

def get_backoff_time(self) -> float:
backoff_value = super().get_backoff_time()
jitter = random.random() * 0.25
backoff_value += jitter
logger.debug(f"Calculating retry backoff: {backoff_value} (jitter: {jitter})")
return backoff_value
1 change: 1 addition & 0 deletions scripts/repl.py
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,7 @@ def cleanup_all(pc):
"delete_all_collections": delete_all_collections,
"delete_all_backups": delete_all_backups,
"cleanup_all": cleanup_all,
"pcl": Pinecone(host="http://localhost:8000"),
# Add any other variables you want to have available in the REPL
}

Expand Down
136 changes: 136 additions & 0 deletions scripts/test-server.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,136 @@
from http.server import BaseHTTPRequestHandler, HTTPServer
import json

backups_response = {
"data": [
{
"backup_id": "6f52240b-6397-481b-9767-748a2d4d3b65",
"source_index_name": "jensparse",
"source_index_id": "71ded150-2b8e-422d-9849-097f2c89d18b",
"status": "Ready",
"cloud": "aws",
"region": "us-east-1",
"tags": {},
"name": "sparsebackup",
"description": "",
"dimension": 0,
"record_count": 10000,
"namespace_count": 1000,
"size_bytes": 123456,
"created_at": "2025-05-15T20:55:29.477794Z",
}
]
}

indexes_response = {
"indexes": [
{
"name": "jhamon-20250515-165135548-reorg-create-with-e",
"metric": "dotproduct",
"host": "jhamon-20250515-165135548-reorg-create-with-e-bt8x3su.svc.aped-4627-b74a.pinecone.io",
"spec": {"serverless": {"cloud": "aws", "region": "us-east-1"}},
"status": {"ready": True, "state": "Ready"},
"vector_type": "sparse",
"dimension": None,
"deletion_protection": "disabled",
"tags": {"env": "dev"},
},
{
"name": "unexpected",
"metric": "newmetric",
"host": "jhamon-20250515-165135548-reorg-create-with-e-bt8x3su.svc.aped-4627-b74a.pinecone.io",
"spec": {"serverless": {"cloud": "aws", "region": "us-east-1"}},
"status": {"ready": False, "state": "UnknownStatus"},
"vector_type": "sparse",
"dimension": -1,
"deletion_protection": "disabled",
"tags": {"env": "dev"},
},
{
"name": "wrong-types",
"metric": 123,
"host": "jhamon-20250515-165135548-reorg-create-with-e-bt8x3su.svc.aped-4627-b74a.pinecone.io",
"spec": {"serverless": {"cloud": "aws", "region": "us-east-1"}},
"status": {"ready": False, "state": "UnknownStatus"},
"vector_type": None,
"dimension": None,
"deletion_protection": "asdf",
"tags": None,
},
]
}

index_description_response = {
"name": "docs-example-dense",
"vector_type": "dense",
"metric": "cosine",
"dimension": 1536,
"status": {"ready": True, "state": "Ready"},
"host": "docs-example-dense-govk0nt.svc.aped-4627-b74a.pinecone.io",
"spec": {"serverless": {"region": "us-east-1", "cloud": "aws"}},
"deletion_protection": "disabled",
"tags": {"environment": "development"},
}

upsert_response = {"upsertedCount": 10}

call_count = 0


class MyHandler(BaseHTTPRequestHandler):
def do_POST(self):
global call_count
call_count += 1

# Simulate a high rate of 500 errors
if call_count % 5 != 0:
self.send_response(500)
self.end_headers()
return

if self.path.startswith("/vectors/upsert"):
self.send_response(200)
self.send_header("Content-type", "application/json")
self.end_headers()
response = upsert_response
self.wfile.write(json.dumps(response).encode())
else:
self.send_response(404)
self.end_headers()

def do_GET(self):
global call_count
call_count += 1

# Simulate a high rate of 500 errors
if call_count % 5 != 0:
self.send_response(500)
self.end_headers()
return

if self.path.startswith("/backups"):
self.send_response(200)
self.send_header("Content-type", "application/json")
self.end_headers()
response = backups_response
self.wfile.write(json.dumps(response).encode())
elif self.path.startswith("/indexes/"):
self.send_response(200)
self.send_header("Content-type", "application/json")
self.end_headers()
response = index_description_response
self.wfile.write(json.dumps(response).encode())
elif self.path.startswith("/indexes"):
self.send_response(200)
self.send_header("Content-type", "application/json")
self.end_headers()
response = indexes_response
self.wfile.write(json.dumps(response).encode())
else:
self.send_response(404)
self.end_headers()


server = HTTPServer(("localhost", 8000), MyHandler)
print("Serving on http://localhost:8000")
server.serve_forever()
49 changes: 49 additions & 0 deletions tests/unit/openapi_support/test_retries.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
import pytest
from unittest.mock import patch, MagicMock
from urllib3.exceptions import MaxRetryError
from urllib3.util.retry import Retry
from pinecone.openapi_support.retries import JitterRetry


def test_jitter_retry_backoff():
"""Test that the backoff time includes jitter."""
retry = JitterRetry(
total=5,
backoff_factor=0.25,
backoff_max=3,
status_forcelist=(500, 502, 503, 504),
allowed_methods=None,
)

# Mock the parent's get_backoff_time to return a fixed value
with patch.object(Retry, "get_backoff_time", return_value=1.0):
# Test multiple times to ensure jitter is added
backoff_times = [retry.get_backoff_time() for _ in range(100)]

# All backoff times should be between 1.0 and 1.25
assert all(1.0 <= t <= 1.25 for t in backoff_times)
# Values should be different (jitter is working)
assert len(set(backoff_times)) > 1


def test_jitter_retry_behavior():
"""Test that retries actually occur and respect the total count."""
retry = JitterRetry(total=3)
mock_response = MagicMock()
mock_response.status = 500 # Simulate server error

# Simulate a failing request
with pytest.raises(MaxRetryError) as exc_info:
retry2 = retry.increment(
method="GET", url="http://test.com", response=mock_response, error=None
)
retry3 = retry2.increment(
method="GET", url="http://test.com", response=mock_response, error=None
)
retry4 = retry3.increment(
method="GET", url="http://test.com", response=mock_response, error=None
)
retry4.increment(method="GET", url="http://test.com", response=mock_response, error=None)

# Verify the error contains the expected information
assert "Max retries exceeded" in str(exc_info.value)