Add retries to tests when waiting on assuming new role via EKS Pod Identity (#713)

*Issue #, if available:* 

*Description of changes:*

Tests have been failing occasionally with the following error:

```
    I0225 13:22:11.375776 22828 credentials.go:1212] Waiting until IAM role for ServiceAccount s3-csi-driver-sa is assumable for EKS Pod Identity (s3-csi-node-9xnvd, 86ff74d5-992b-4ae5-a4e2-6c3de483636a, kube-system)
    I0225 13:22:11.524430 22828 credentials.go:1156] Unexpected error: 
        <*smithy.OperationError | 0xc000ef2690>: 
        operation error EKS Auth: AssumeRoleForPodIdentity, https response error StatusCode: 404, RequestID: 4a91f663-7fbf-46d1-9188-5c63cccaa7ad, ResourceNotFoundException: The token included in the request has no service account role association for it.
```

This error implies that the EKS Auth service has not yet registered the
new assocation.

This change adds `ResourceNotFoundException` to the list of error codes
that will trigger the SDK to retry in the function when waiting for the
role to be "assumable".

By submitting this pull request, I confirm that you can use, modify,
copy, and redistribute this contribution, under the terms of your
choice.

Signed-off-by: Daniel Carl Jones <djonesoa@amazon.com>

Author: dannycjones

tests/e2e-kubernetes/testsuites/credentials.go

@@ -58,11 +58,14 @@ const (
 )
 
 const (
-	eksauthAssumeRoleRetryCode            = "AccessDeniedException"
 	eksauthAssumeRoleRetryMaxAttempts     = 0 // This will cause SDK to retry indefinitely, but we do have a timeout on the operation
 	eksauthAssumeRoleRetryMaxBackoffDelay = 10 * time.Second
 )
 
+var (
+	eksauthAssumeRoleRetryErrorCodes = []string{"AccessDeniedException", "ResourceNotFoundException"}
+)
+
 const (
 	iamListAttachedRolePoliciesTimeout = 1 * time.Minute
 	iamListAttachedRolePoliciesPolling = 5 * time.Second
@@ -1139,9 +1142,10 @@ func assumeRole(ctx context.Context, f *framework.Framework, roleArn string) *st
 	})
 }
 
-// waitUntilRoleIsAssumable waits until the given role is assumable.
+// Waits until the given role is assumable.
+//
 // This is needed because we're creating new roles in our test cases and then trying to assume those roles,
-// but there is a delay between IAM and STS services and newly created roles/policies does not appear on STS immediately.
+// but there's a delay between IAM and the token service (STS or EKS Auth) resulting in errors such as "access denied" or "not found".
 func waitUntilRoleIsAssumable[Input any, Output any, O any](
 	ctx context.Context,
 	assumeFunc func(context.Context, *Input, ...func(O)) (*Output, error),
@@ -1176,7 +1180,7 @@ func waitUntilRoleIsAssumableEKS[Input any, Output any](
 	input *Input,
 ) *Output {
 	return waitUntilRoleIsAssumable(ctx, assumeFunc, input, func(o *eksauth.Options) {
-		o.Retryer = retry.AddWithErrorCodes(o.Retryer, eksauthAssumeRoleRetryCode)
+		o.Retryer = retry.AddWithErrorCodes(o.Retryer, eksauthAssumeRoleRetryErrorCodes...)
 		o.Retryer = retry.AddWithMaxAttempts(o.Retryer, eksauthAssumeRoleRetryMaxAttempts)
 		o.Retryer = retry.AddWithMaxBackoffDelay(o.Retryer, eksauthAssumeRoleRetryMaxBackoffDelay)
 	})
@@ -1204,7 +1208,7 @@ func waitUntilRoleIsAssumableWithWebIdentity(ctx context.Context, f *framework.F
 }
 
 func waitUntilRoleIsAssumableWithEKS(ctx context.Context, f *framework.Framework, sa *v1.ServiceAccount, pod *v1.Pod) {
-	// If you're seeing the following error, then it means you've made a typo in the cluster name when running the tests!
+	// If you see the following error, it may mean you've made a typo in the cluster name or the role is being assumed too quickly.
 	// [FAILED] operation error EKS Auth: AssumeRoleForPodIdentity, https response error StatusCode: 404, RequestID:
 	// ResourceNotFoundException: The token included in the request has no service account role association for it.