Skip to content

Commit e36979c

Browse files
kieranbrownkieranbrown
committed
Make EcsCredentialProvider retry behavior configurable
1 parent d15e6bc commit e36979c

2 files changed

Lines changed: 225 additions & 1 deletion

File tree

‎src/Credentials/EcsCredentialProvider.php‎

Lines changed: 42 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,11 +41,22 @@ class EcsCredentialProvider
4141
/** @var int */
4242
private $attempts;
4343

44+
/** @var string[] */
45+
private $retryableExceptions;
46+
47+
/** @var int[] */
48+
private $retryableErrorCodes;
49+
4450
/**
4551
* The constructor accepts following options:
4652
* - timeout: (optional) Connection timeout, in seconds, default 1.0
4753
* - retries: Optional number of retries to be attempted, default 3.
4854
* - client: An EcsClient to make request from
55+
* - retryable_exceptions: Optional array of additional exception class
56+
* names that should be retried. Connection errors are always retried,
57+
* regardless of this option.
58+
* - retryable_error_codes: Optional array of HTTP status codes that
59+
* should be retried. Defaults to an empty array.
4960
*
5061
* @param array $config Configuration options
5162
*/
@@ -58,6 +69,8 @@ public function __construct(array $config = [])
5869
: ((int) getenv(self::ENV_RETRIES) ?: self::DEFAULT_ENV_RETRIES);
5970

6071
$this->client = $config['client'] ?? \Aws\default_http_handler();
72+
$this->retryableExceptions = $config['retryable_exceptions'] ?? [];
73+
$this->retryableErrorCodes = $config['retryable_error_codes'] ?? [];
6174
}
6275

6376
/**
@@ -106,7 +119,8 @@ public function __invoke()
106119
})->otherwise(function ($reason) {
107120
$connectionError = is_array($reason) && !empty($reason['connection_error']);
108121
$exception = is_array($reason) ? ($reason['exception'] ?? null) : $reason;
109-
$isRetryable = $connectionError || ($exception instanceof \Throwable && HttpHandlerError::isConnectionError($exception));
122+
$isRetryable = $connectionError
123+
|| ($exception instanceof \Throwable && $this->isRetryable($exception));
110124

111125
if ($isRetryable && ($this->attempts < $this->retries)) {
112126
sleep((int)pow(1.2, $this->attempts));
@@ -221,6 +235,33 @@ private function getEcsUri()
221235
return self::SERVER_URI . $credsUri;
222236
}
223237

238+
/**
239+
* Determines whether a failed request should be retried. Connection
240+
* errors are always retried; the configured retryable_exceptions and
241+
* retryable_error_codes are checked in addition to them.
242+
*/
243+
private function isRetryable(\Throwable $exception): bool
244+
{
245+
if (HttpHandlerError::isConnectionError($exception)) {
246+
return true;
247+
}
248+
249+
foreach ($this->retryableExceptions as $exceptionClass) {
250+
if ($exception instanceof $exceptionClass) {
251+
return true;
252+
}
253+
}
254+
255+
$response = HttpHandlerError::getResponse($exception);
256+
257+
return $response !== null
258+
&& in_array(
259+
$response->getStatusCode(),
260+
$this->retryableErrorCodes,
261+
true
262+
);
263+
}
264+
224265
private function decodeResult($response)
225266
{
226267
$result = json_decode($response, true);

‎tests/Credentials/EcsCredentialProviderTest.php‎

Lines changed: 183 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -462,6 +462,15 @@ public static function successDataProvider(): array
462462
]);
463463
$rejectionRawConnectException = Promise\Create::rejectionFor($connectException);
464464

465+
$tooManyRequestsException = self::createRequestException(
466+
'429 Too Many Requests',
467+
new Psr7\Request('GET', '/latest'),
468+
new Psr7\Response(429)
469+
);
470+
$rejectionTooManyRequests = Promise\Create::rejectionFor([
471+
'exception' => $tooManyRequestsException,
472+
]);
473+
465474
$promiseCreds = Promise\Create::promiseFor(
466475
new Response(200, [], Psr7\Utils::streamFor(
467476
json_encode(call_user_func_array(
@@ -524,6 +533,135 @@ public static function successDataProvider(): array
524533
];
525534
}
526535

536+
public function testRetriesOptedInErrorCode()
537+
{
538+
$expiry = time() + 1000;
539+
$creds = ['foo_key', 'baz_secret', 'qux_token', "@{$expiry}"];
540+
541+
$rejectionTooManyRequests = Promise\Create::rejectionFor([
542+
'exception' => self::createRequestException(
543+
'429 Too Many Requests',
544+
new Psr7\Request('GET', '/latest'),
545+
new Psr7\Response(429)
546+
),
547+
]);
548+
$promiseCreds = Promise\Create::promiseFor(
549+
new Response(200, [], Psr7\Utils::streamFor(
550+
json_encode(call_user_func_array(
551+
[self::class, 'getCredentialArray'],
552+
$creds
553+
)))
554+
)
555+
);
556+
557+
$provider = new EcsCredentialProvider([
558+
'client' => $this->getTestClient([
559+
$rejectionTooManyRequests,
560+
$promiseCreds,
561+
], $creds),
562+
'retries' => 2,
563+
'retryable_error_codes' => [429],
564+
]);
565+
566+
$credentials = $provider()->wait();
567+
$this->assertSame('foo_key', $credentials->getAccessKeyId());
568+
$this->assertSame('baz_secret', $credentials->getSecretKey());
569+
}
570+
571+
public function testDoesNotRetry429ByDefault()
572+
{
573+
$rejectionTooManyRequests = Promise\Create::rejectionFor([
574+
'exception' => self::createRequestException(
575+
'429 Too Many Requests',
576+
new Psr7\Request('GET', '/latest'),
577+
new Psr7\Response(429)
578+
),
579+
]);
580+
581+
$provider = new EcsCredentialProvider([
582+
'client' => $this->getTestClient([
583+
$rejectionTooManyRequests,
584+
]),
585+
'retries' => 3,
586+
]);
587+
588+
try {
589+
$provider()->wait();
590+
$this->fail('Provider should have thrown an exception.');
591+
} catch (CredentialsException $e) {
592+
$this->assertStringContainsString(
593+
'attempt 0/3',
594+
$e->getMessage()
595+
);
596+
$this->assertStringContainsString('429 Too Many Requests', $e->getMessage());
597+
}
598+
599+
$this->assertSame(0, $provider->getAttempts());
600+
}
601+
602+
public function testRetriesOptedInExceptionClass()
603+
{
604+
$expiry = time() + 1000;
605+
$creds = ['foo_key', 'baz_secret', 'qux_token', "@{$expiry}"];
606+
607+
$rejectionRequest = Promise\Create::rejectionFor([
608+
'exception' => new \DomainException('Boom'),
609+
]);
610+
$promiseCreds = Promise\Create::promiseFor(
611+
new Response(200, [], Psr7\Utils::streamFor(
612+
json_encode(call_user_func_array(
613+
[self::class, 'getCredentialArray'],
614+
$creds
615+
)))
616+
)
617+
);
618+
619+
$provider = new EcsCredentialProvider([
620+
'client' => $this->getTestClient([
621+
$rejectionRequest,
622+
$promiseCreds,
623+
], $creds),
624+
'retries' => 2,
625+
'retryable_exceptions' => [\DomainException::class],
626+
]);
627+
628+
$credentials = $provider()->wait();
629+
$this->assertSame('foo_key', $credentials->getAccessKeyId());
630+
}
631+
632+
public function testCustomRetryableExceptionsAreAddedToDefaults()
633+
{
634+
$expiry = time() + 1000;
635+
$creds = ['foo_key', 'baz_secret', 'qux_token', "@{$expiry}"];
636+
637+
$rejectionConnection = Promise\Create::rejectionFor([
638+
'exception' => new ConnectException(
639+
'cURL error 28: Connection timed out after 1000 milliseconds',
640+
new Psr7\Request('GET', '/latest')
641+
),
642+
]);
643+
$promiseCreds = Promise\Create::promiseFor(
644+
new Response(200, [], Psr7\Utils::streamFor(
645+
json_encode(call_user_func_array(
646+
[self::class, 'getCredentialArray'],
647+
$creds
648+
)))
649+
)
650+
);
651+
652+
$provider = new EcsCredentialProvider([
653+
'client' => $this->getTestClient([
654+
$rejectionConnection,
655+
$promiseCreds,
656+
], $creds),
657+
'retries' => 2,
658+
'retryable_exceptions' => [\DomainException::class],
659+
]);
660+
661+
$credentials = $provider()->wait();
662+
$this->assertSame('foo_key', $credentials->getAccessKeyId());
663+
}
664+
527665
/**
528666
* @param $client
529667
* @param \Exception $expected
@@ -566,6 +704,13 @@ public static function failureDataProvider(): array
566704
'connection_error' => true,
567705
'exception' => new \Exception('cURL error 28: Connection timed out after 1000 milliseconds'),
568706
]);
707+
$rejectionTooManyRequests = Promise\Create::rejectionFor([
708+
'exception' => self::createRequestException(
709+
'429 Too Many Requests',
710+
$getRequest,
711+
new Psr7\Response(429)
712+
)
713+
]);
569714

570715
return [
571716
'Non-retryable error' => [
@@ -585,9 +730,47 @@ public static function failureDataProvider(): array
585730
'Error retrieving credentials from container metadata after attempt 1/1 (cURL error 28: Connection timed out after 1000 milliseconds)'
586731
)
587732
],
733+
'Non-retryable HTTP 429 by default' => [
734+
[
735+
$rejectionTooManyRequests,
736+
],
737+
new CredentialsException(
738+
'Error retrieving credentials from container metadata after attempt 0/1 (429 Too Many Requests)'
739+
)
740+
],
588741
];
589742
}
590743

744+
public function testOptedInHTTP429RetryExhaustsAttempts()
745+
{
746+
$rejectionTooManyRequests = Promise\Create::rejectionFor([
747+
'exception' => self::createRequestException(
748+
'429 Too Many Requests',
749+
new Psr7\Request('GET', '/latest'),
750+
new Psr7\Response(429)
751+
),
752+
]);
753+
754+
$provider = new EcsCredentialProvider([
755+
'client' => $this->getTestClient([
756+
$rejectionTooManyRequests,
757+
$rejectionTooManyRequests,
758+
]),
759+
'retries' => 1,
760+
'retryable_error_codes' => [429],
761+
]);
762+
763+
try {
764+
$provider()->wait();
765+
$this->fail('Provider should have thrown an exception.');
766+
} catch (CredentialsException $e) {
767+
$this->assertSame(
768+
'Error retrieving credentials from container metadata after attempt 1/1 (429 Too Many Requests)',
769+
$e->getMessage()
770+
);
771+
}
772+
}
773+
591774
public function testReadsRetriesFromEnvironment()
592775
{
593776
putenv('AWS_METADATA_SERVICE_NUM_ATTEMPTS=1');

0 commit comments

Comments
 (0)