Add retry config for urllib3 requests (#491)

## Problem

We want to automatically retry when errors occur

## Solution

Implement `urllib3` retry configuration. We implemented the backup
calculation with jitter ourselves because this is not available for all
versions of `urllib3` that the SDK uses.

## Type of Change

- [x] New feature (non-breaking change which adds functionality)

## Test Plan

I created a mock server in `scripts/text-server.py` that simulates a
high rate of failures (80% failure, only 1 in 5 requests succeed).

`poetry run python3 scripts/test-server.py`

Then I made some requests and observed logging to see what was going on.

```python
>>>  from pinecone import Pinecone
>>>  # Testing control plane with retries
>>> pc = Pinecone(host='http://localhost:8000')
>>> pc.list_indexes()
>>> 
>>> # Data plane
>>> idx = pc.Index(host='http://localhost:8000')
>>> # enable debug logging
>>> idx._vector_api.api_client.configuration.debug = True
>>> 
>>> idx.upsert(vectors=[('1', [0.1, 0.2])])
DEBUG    | pinecone.openapi_support.rest_urllib3:126 | Calling urllib3 request()
DEBUG    | urllib3.connectionpool:546 | http://localhost:8000 "POST /vectors/upsert HTTP/10" 500 None
DEBUG    | urllib3.util.retry:521 | Incremented Retry for (url='/vectors/upsert'): JitterRetry(total=4, connect=None, read=None, redirect=None, status=None)
DEBUG    | pinecone.openapi_support.retries:20 | Calculating retry backoff: 0.15197003184454544 (jitter: 0.15197003184454544)
DEBUG    | urllib3.connectionpool:943 | Retry: /vectors/upsert
DEBUG    | urllib3.connectionpool:546 | http://localhost:8000 "POST /vectors/upsert HTTP/10" 500 None
DEBUG    | urllib3.util.retry:521 | Incremented Retry for (url='/vectors/upsert'): JitterRetry(total=3, connect=None, read=None, redirect=None, status=None)
DEBUG    | pinecone.openapi_support.retries:20 | Calculating retry backoff: 0.7352149950424516 (jitter: 0.2352149950424516)
DEBUG    | urllib3.connectionpool:943 | Retry: /vectors/upsert
DEBUG    | urllib3.connectionpool:546 | http://localhost:8000 "POST /vectors/upsert HTTP/10" 500 None
DEBUG    | urllib3.util.retry:521 | Incremented Retry for (url='/vectors/upsert'): JitterRetry(total=2, connect=None, read=None, redirect=None, status=None)
DEBUG    | pinecone.openapi_support.retries:20 | Calculating retry backoff: 1.1307109027442626 (jitter: 0.13071090274426245)
DEBUG    | urllib3.connectionpool:943 | Retry: /vectors/upsert
DEBUG    | urllib3.connectionpool:546 | http://localhost:8000 "POST /vectors/upsert HTTP/10" 500 None
DEBUG    | urllib3.util.retry:521 | Incremented Retry for (url='/vectors/upsert'): JitterRetry(total=1, connect=None, read=None, redirect=None, status=None)
DEBUG    | pinecone.openapi_support.retries:20 | Calculating retry backoff: 2.142226695165083 (jitter: 0.14222669516508277)
DEBUG    | urllib3.connectionpool:943 | Retry: /vectors/upsert
DEBUG    | urllib3.connectionpool:546 | http://localhost:8000 "POST /vectors/upsert HTTP/10" 200 None
DEBUG    | pinecone.openapi_support.rest_urllib3:266 | response body: b'{"upsertedCount": 10}'
DEBUG    | pinecone.openapi_support.rest_utils:34 | response status: 200
{'upserted_count': 10}

```

Author: jhamon

pinecone/openapi_support/rest_urllib3.py

@@ -8,7 +8,7 @@
 from .rest_utils import raise_exceptions_or_return, RESTResponse, RestClientInterface
 
 import urllib3
-
+from .retries import JitterRetry
 from .exceptions import PineconeApiException, PineconeApiValueError
 
 
@@ -52,6 +52,13 @@ def __init__(
 
         if configuration.retries is not None:
             addition_pool_args["retries"] = configuration.retries
+        else:
+            addition_pool_args["retries"] = JitterRetry(
+                total=5,
+                backoff_factor=0.25,
+                status_forcelist=(500, 502, 503, 504),
+                allowed_methods=None,
+            )
 
         if configuration.socket_options is not None:
             addition_pool_args["socket_options"] = configuration.socket_options

pinecone/openapi_support/retries.py

@@ -0,0 +1,21 @@
+import random
+from urllib3.util.retry import Retry
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class JitterRetry(Retry):
+    """
+    Retry with exponential back‑off with jitter.
+
+    The Retry class is being extended as built-in support for jitter was added only in urllib3 2.0.0.
+    Jitter logic is following the official implementation with a constant jitter factor: https://github.com/urllib3/urllib3/blob/main/src/urllib3/util/retry.py
+    """
+
+    def get_backoff_time(self) -> float:
+        backoff_value = super().get_backoff_time()
+        jitter = random.random() * 0.25
+        backoff_value += jitter
+        logger.debug(f"Calculating retry backoff: {backoff_value} (jitter: {jitter})")
+        return backoff_value

scripts/repl.py

@@ -98,6 +98,7 @@ def cleanup_all(pc):
         "delete_all_collections": delete_all_collections,
         "delete_all_backups": delete_all_backups,
         "cleanup_all": cleanup_all,
+        "pcl": Pinecone(host="http://localhost:8000"),
         # Add any other variables you want to have available in the REPL
     }
 

scripts/test-server.py

@@ -0,0 +1,136 @@
+from http.server import BaseHTTPRequestHandler, HTTPServer
+import json
+
+backups_response = {
+    "data": [
+        {
+            "backup_id": "6f52240b-6397-481b-9767-748a2d4d3b65",
+            "source_index_name": "jensparse",
+            "source_index_id": "71ded150-2b8e-422d-9849-097f2c89d18b",
+            "status": "Ready",
+            "cloud": "aws",
+            "region": "us-east-1",
+            "tags": {},
+            "name": "sparsebackup",
+            "description": "",
+            "dimension": 0,
+            "record_count": 10000,
+            "namespace_count": 1000,
+            "size_bytes": 123456,
+            "created_at": "2025-05-15T20:55:29.477794Z",
+        }
+    ]
+}
+
+indexes_response = {
+    "indexes": [
+        {
+            "name": "jhamon-20250515-165135548-reorg-create-with-e",
+            "metric": "dotproduct",
+            "host": "jhamon-20250515-165135548-reorg-create-with-e-bt8x3su.svc.aped-4627-b74a.pinecone.io",
+            "spec": {"serverless": {"cloud": "aws", "region": "us-east-1"}},
+            "status": {"ready": True, "state": "Ready"},
+            "vector_type": "sparse",
+            "dimension": None,
+            "deletion_protection": "disabled",
+            "tags": {"env": "dev"},
+        },
+        {
+            "name": "unexpected",
+            "metric": "newmetric",
+            "host": "jhamon-20250515-165135548-reorg-create-with-e-bt8x3su.svc.aped-4627-b74a.pinecone.io",
+            "spec": {"serverless": {"cloud": "aws", "region": "us-east-1"}},
+            "status": {"ready": False, "state": "UnknownStatus"},
+            "vector_type": "sparse",
+            "dimension": -1,
+            "deletion_protection": "disabled",
+            "tags": {"env": "dev"},
+        },
+        {
+            "name": "wrong-types",
+            "metric": 123,
+            "host": "jhamon-20250515-165135548-reorg-create-with-e-bt8x3su.svc.aped-4627-b74a.pinecone.io",
+            "spec": {"serverless": {"cloud": "aws", "region": "us-east-1"}},
+            "status": {"ready": False, "state": "UnknownStatus"},
+            "vector_type": None,
+            "dimension": None,
+            "deletion_protection": "asdf",
+            "tags": None,
+        },
+    ]
+}
+
+index_description_response = {
+    "name": "docs-example-dense",
+    "vector_type": "dense",
+    "metric": "cosine",
+    "dimension": 1536,
+    "status": {"ready": True, "state": "Ready"},
+    "host": "docs-example-dense-govk0nt.svc.aped-4627-b74a.pinecone.io",
+    "spec": {"serverless": {"region": "us-east-1", "cloud": "aws"}},
+    "deletion_protection": "disabled",
+    "tags": {"environment": "development"},
+}
+
+upsert_response = {"upsertedCount": 10}
+
+call_count = 0
+
+
+class MyHandler(BaseHTTPRequestHandler):
+    def do_POST(self):
+        global call_count
+        call_count += 1
+
+        # Simulate a high rate of 500 errors
+        if call_count % 5 != 0:
+            self.send_response(500)
+            self.end_headers()
+            return
+
+        if self.path.startswith("/vectors/upsert"):
+            self.send_response(200)
+            self.send_header("Content-type", "application/json")
+            self.end_headers()
+            response = upsert_response
+            self.wfile.write(json.dumps(response).encode())
+        else:
+            self.send_response(404)
+            self.end_headers()
+
+    def do_GET(self):
+        global call_count
+        call_count += 1
+
+        # Simulate a high rate of 500 errors
+        if call_count % 5 != 0:
+            self.send_response(500)
+            self.end_headers()
+            return
+
+        if self.path.startswith("/backups"):
+            self.send_response(200)
+            self.send_header("Content-type", "application/json")
+            self.end_headers()
+            response = backups_response
+            self.wfile.write(json.dumps(response).encode())
+        elif self.path.startswith("/indexes/"):
+            self.send_response(200)
+            self.send_header("Content-type", "application/json")
+            self.end_headers()
+            response = index_description_response
+            self.wfile.write(json.dumps(response).encode())
+        elif self.path.startswith("/indexes"):
+            self.send_response(200)
+            self.send_header("Content-type", "application/json")
+            self.end_headers()
+            response = indexes_response
+            self.wfile.write(json.dumps(response).encode())
+        else:
+            self.send_response(404)
+            self.end_headers()
+
+
+server = HTTPServer(("localhost", 8000), MyHandler)
+print("Serving on http://localhost:8000")
+server.serve_forever()

tests/unit/openapi_support/test_retries.py

@@ -0,0 +1,49 @@
+import pytest
+from unittest.mock import patch, MagicMock
+from urllib3.exceptions import MaxRetryError
+from urllib3.util.retry import Retry
+from pinecone.openapi_support.retries import JitterRetry
+
+
+def test_jitter_retry_backoff():
+    """Test that the backoff time includes jitter."""
+    retry = JitterRetry(
+        total=5,
+        backoff_factor=0.25,
+        backoff_max=3,
+        status_forcelist=(500, 502, 503, 504),
+        allowed_methods=None,
+    )
+
+    # Mock the parent's get_backoff_time to return a fixed value
+    with patch.object(Retry, "get_backoff_time", return_value=1.0):
+        # Test multiple times to ensure jitter is added
+        backoff_times = [retry.get_backoff_time() for _ in range(100)]
+
+        # All backoff times should be between 1.0 and 1.25
+        assert all(1.0 <= t <= 1.25 for t in backoff_times)
+        # Values should be different (jitter is working)
+        assert len(set(backoff_times)) > 1
+
+
+def test_jitter_retry_behavior():
+    """Test that retries actually occur and respect the total count."""
+    retry = JitterRetry(total=3)
+    mock_response = MagicMock()
+    mock_response.status = 500  # Simulate server error
+
+    # Simulate a failing request
+    with pytest.raises(MaxRetryError) as exc_info:
+        retry2 = retry.increment(
+            method="GET", url="http://test.com", response=mock_response, error=None
+        )
+        retry3 = retry2.increment(
+            method="GET", url="http://test.com", response=mock_response, error=None
+        )
+        retry4 = retry3.increment(
+            method="GET", url="http://test.com", response=mock_response, error=None
+        )
+        retry4.increment(method="GET", url="http://test.com", response=mock_response, error=None)
+
+    # Verify the error contains the expected information
+    assert "Max retries exceeded" in str(exc_info.value)