diff --git a/pinecone/openapi_support/rest_urllib3.py b/pinecone/openapi_support/rest_urllib3.py index 0c1a1c5a..f310ca99 100644 --- a/pinecone/openapi_support/rest_urllib3.py +++ b/pinecone/openapi_support/rest_urllib3.py @@ -8,7 +8,7 @@ from .rest_utils import raise_exceptions_or_return, RESTResponse, RestClientInterface import urllib3 - +from .retries import JitterRetry from .exceptions import PineconeApiException, PineconeApiValueError @@ -52,6 +52,13 @@ def __init__( if configuration.retries is not None: addition_pool_args["retries"] = configuration.retries + else: + addition_pool_args["retries"] = JitterRetry( + total=5, + backoff_factor=0.25, + status_forcelist=(500, 502, 503, 504), + allowed_methods=None, + ) if configuration.socket_options is not None: addition_pool_args["socket_options"] = configuration.socket_options diff --git a/pinecone/openapi_support/retries.py b/pinecone/openapi_support/retries.py new file mode 100644 index 00000000..2b91a31d --- /dev/null +++ b/pinecone/openapi_support/retries.py @@ -0,0 +1,21 @@ +import random +from urllib3.util.retry import Retry +import logging + +logger = logging.getLogger(__name__) + + +class JitterRetry(Retry): + """ + Retry with exponential back‑off with jitter. + + The Retry class is being extended as built-in support for jitter was added only in urllib3 2.0.0. + Jitter logic is following the official implementation with a constant jitter factor: https://github.com/urllib3/urllib3/blob/main/src/urllib3/util/retry.py + """ + + def get_backoff_time(self) -> float: + backoff_value = super().get_backoff_time() + jitter = random.random() * 0.25 + backoff_value += jitter + logger.debug(f"Calculating retry backoff: {backoff_value} (jitter: {jitter})") + return backoff_value diff --git a/scripts/repl.py b/scripts/repl.py index 88c218b2..f9f85000 100644 --- a/scripts/repl.py +++ b/scripts/repl.py @@ -88,6 +88,7 @@ def cleanup_all(pc): "delete_all_collections": delete_all_collections, "delete_all_backups": delete_all_backups, "cleanup_all": cleanup_all, + "pcl": Pinecone(host="http://localhost:8000"), # Add any other variables you want to have available in the REPL } diff --git a/scripts/test-server.py b/scripts/test-server.py new file mode 100644 index 00000000..784d510d --- /dev/null +++ b/scripts/test-server.py @@ -0,0 +1,136 @@ +from http.server import BaseHTTPRequestHandler, HTTPServer +import json + +backups_response = { + "data": [ + { + "backup_id": "6f52240b-6397-481b-9767-748a2d4d3b65", + "source_index_name": "jensparse", + "source_index_id": "71ded150-2b8e-422d-9849-097f2c89d18b", + "status": "Ready", + "cloud": "aws", + "region": "us-east-1", + "tags": {}, + "name": "sparsebackup", + "description": "", + "dimension": 0, + "record_count": 10000, + "namespace_count": 1000, + "size_bytes": 123456, + "created_at": "2025-05-15T20:55:29.477794Z", + } + ] +} + +indexes_response = { + "indexes": [ + { + "name": "jhamon-20250515-165135548-reorg-create-with-e", + "metric": "dotproduct", + "host": "jhamon-20250515-165135548-reorg-create-with-e-bt8x3su.svc.aped-4627-b74a.pinecone.io", + "spec": {"serverless": {"cloud": "aws", "region": "us-east-1"}}, + "status": {"ready": True, "state": "Ready"}, + "vector_type": "sparse", + "dimension": None, + "deletion_protection": "disabled", + "tags": {"env": "dev"}, + }, + { + "name": "unexpected", + "metric": "newmetric", + "host": "jhamon-20250515-165135548-reorg-create-with-e-bt8x3su.svc.aped-4627-b74a.pinecone.io", + "spec": {"serverless": {"cloud": "aws", "region": "us-east-1"}}, + "status": {"ready": False, "state": "UnknownStatus"}, + "vector_type": "sparse", + "dimension": -1, + "deletion_protection": "disabled", + "tags": {"env": "dev"}, + }, + { + "name": "wrong-types", + "metric": 123, + "host": "jhamon-20250515-165135548-reorg-create-with-e-bt8x3su.svc.aped-4627-b74a.pinecone.io", + "spec": {"serverless": {"cloud": "aws", "region": "us-east-1"}}, + "status": {"ready": False, "state": "UnknownStatus"}, + "vector_type": None, + "dimension": None, + "deletion_protection": "asdf", + "tags": None, + }, + ] +} + +index_description_response = { + "name": "docs-example-dense", + "vector_type": "dense", + "metric": "cosine", + "dimension": 1536, + "status": {"ready": True, "state": "Ready"}, + "host": "docs-example-dense-govk0nt.svc.aped-4627-b74a.pinecone.io", + "spec": {"serverless": {"region": "us-east-1", "cloud": "aws"}}, + "deletion_protection": "disabled", + "tags": {"environment": "development"}, +} + +upsert_response = {"upsertedCount": 10} + +call_count = 0 + + +class MyHandler(BaseHTTPRequestHandler): + def do_POST(self): + global call_count + call_count += 1 + + # Simulate a high rate of 500 errors + if call_count % 5 != 0: + self.send_response(500) + self.end_headers() + return + + if self.path.startswith("/vectors/upsert"): + self.send_response(200) + self.send_header("Content-type", "application/json") + self.end_headers() + response = upsert_response + self.wfile.write(json.dumps(response).encode()) + else: + self.send_response(404) + self.end_headers() + + def do_GET(self): + global call_count + call_count += 1 + + # Simulate a high rate of 500 errors + if call_count % 5 != 0: + self.send_response(500) + self.end_headers() + return + + if self.path.startswith("/backups"): + self.send_response(200) + self.send_header("Content-type", "application/json") + self.end_headers() + response = backups_response + self.wfile.write(json.dumps(response).encode()) + elif self.path.startswith("/indexes/"): + self.send_response(200) + self.send_header("Content-type", "application/json") + self.end_headers() + response = index_description_response + self.wfile.write(json.dumps(response).encode()) + elif self.path.startswith("/indexes"): + self.send_response(200) + self.send_header("Content-type", "application/json") + self.end_headers() + response = indexes_response + self.wfile.write(json.dumps(response).encode()) + else: + self.send_response(404) + self.end_headers() + + +server = HTTPServer(("localhost", 8000), MyHandler) +print("Serving on http://localhost:8000") +server.serve_forever() diff --git a/tests/unit/openapi_support/test_retries.py b/tests/unit/openapi_support/test_retries.py new file mode 100644 index 00000000..5f31221d --- /dev/null +++ b/tests/unit/openapi_support/test_retries.py @@ -0,0 +1,49 @@ +import pytest +from unittest.mock import patch, MagicMock +from urllib3.exceptions import MaxRetryError +from urllib3.util.retry import Retry +from pinecone.openapi_support.retries import JitterRetry + + +def test_jitter_retry_backoff(): + """Test that the backoff time includes jitter.""" + retry = JitterRetry( + total=5, + backoff_factor=0.25, + backoff_max=3, + status_forcelist=(500, 502, 503, 504), + allowed_methods=None, + ) + + # Mock the parent's get_backoff_time to return a fixed value + with patch.object(Retry, "get_backoff_time", return_value=1.0): + # Test multiple times to ensure jitter is added + backoff_times = [retry.get_backoff_time() for _ in range(100)] + + # All backoff times should be between 1.0 and 1.25 + assert all(1.0 <= t <= 1.25 for t in backoff_times) + # Values should be different (jitter is working) + assert len(set(backoff_times)) > 1 + + +def test_jitter_retry_behavior(): + """Test that retries actually occur and respect the total count.""" + retry = JitterRetry(total=3) + mock_response = MagicMock() + mock_response.status = 500 # Simulate server error + + # Simulate a failing request + with pytest.raises(MaxRetryError) as exc_info: + retry2 = retry.increment( + method="GET", url="http://test.com", response=mock_response, error=None + ) + retry3 = retry2.increment( + method="GET", url="http://test.com", response=mock_response, error=None + ) + retry4 = retry3.increment( + method="GET", url="http://test.com", response=mock_response, error=None + ) + retry4.increment(method="GET", url="http://test.com", response=mock_response, error=None) + + # Verify the error contains the expected information + assert "Max retries exceeded" in str(exc_info.value)