EKS Pod Identity support for pod level credentials (#458)

*Description of changes:* With this change, we will support the
configuration of pod level credentials with EKS Pod Identity. In EKS
clusters, this is an alternative to IRSA with an easier configuration
process.

*Key Changes:*

- Credential Provider
- In `provider_pod.go` the precedence is given to IRSA credentials. If
they are not configured, then EKS Pod Identity credentials are forwarded
to the Mountpoint process instead.
  - Added unit tests

- Testing
- Added E2E tests covering EKS Pod Identity setup scenarios with IAM
roles of varying levels of S3 access
- Added one pod level IRSA E2E test to make sure driver level EKS Pod
Identity is not used instead
  - Added E2E tests covering setup scenario of IRSA and EKS Pod Identity
  - Manual testing performed on personal EKS cluster

By submitting this pull request, I confirm that you can use, modify,
copy, and redistribute this contribution, under the terms of your
choice.

---------

Signed-off-by: Renan Magagnin <renanmag@amazon.co.uk>

Author: renanmagagnin

charts/aws-mountpoint-s3-csi-driver/templates/csidriver.yaml

@@ -12,4 +12,8 @@ spec:
   tokenRequests:
     - audience: "sts.amazonaws.com"
       expirationSeconds: 3600
+    {{- if .Values.experimental.podMounter }}
+    - audience: "pods.eks.amazonaws.com"
+      expirationSeconds: 3600
+    {{- end }}
   requiresRepublish: true

pkg/driver/node/credentialprovider/provider_pod.go

@@ -16,11 +16,17 @@ import (
 	"k8s.io/klog/v2"
 
 	"github.com/awslabs/aws-s3-csi-driver/pkg/driver/node/envprovider"
+	"github.com/awslabs/aws-s3-csi-driver/pkg/util"
 )
 
 const (
-	serviceAccountTokenAudienceSTS = "sts.amazonaws.com"
-	serviceAccountRoleAnnotation   = "eks.amazonaws.com/role-arn"
+	serviceAccountTokenAudienceSTS         = "sts.amazonaws.com"
+	serviceAccountTokenAudiencePodIdentity = "pods.eks.amazonaws.com"
+	serviceAccountRoleAnnotation           = "eks.amazonaws.com/role-arn"
+	// TODO: Add a driver configuration flag to handle custom values of podIdentityCredURI. Currently we are assuming the default IPv4 address as determined in the references below:
+	// Doc: https://docs.aws.amazon.com/eks/latest/userguide/pod-id-agent-setup.html
+	// Source code: https://github.com/aws/eks-pod-identity-agent/blob/8bd71a236522993f02427083e485c83f6ae4fe31/configuration/config.go
+	podIdentityCredURI = "http://169.254.170.23/v1/credentials"
 )
 
 const podLevelCredentialsDocsPage = "https://github.com/awslabs/mountpoint-s3-csi-driver/blob/main/docs/CONFIGURATION.md#pod-level-credentials"
@@ -35,6 +41,12 @@ type serviceAccountToken struct {
 func (c *Provider) provideFromPod(ctx context.Context, provideCtx ProvideContext) (envprovider.Environment, error) {
 	klog.V(4).Infof("credentialprovider: Using pod identity")
 
+	podID := provideCtx.GetCredentialPodID()
+	if podID == "" {
+		return nil, status.Error(codes.InvalidArgument, "Missing Pod info. Please make sure to enable `podInfoOnMountCompat`, see "+podLevelCredentialsDocsPage)
+	}
+
+	// 1. Parse ServiceAccountTokens map
 	tokensJson := provideCtx.ServiceAccountTokens
 	if tokensJson == "" {
 		klog.Error("credentialprovider: `authenticationSource` configured to `pod` but no service account tokens are received. Please make sure to enable `podInfoOnMountCompat`, see " + podLevelCredentialsDocsPage)
@@ -48,78 +60,110 @@ func (c *Provider) provideFromPod(ctx context.Context, provideCtx ProvideContext
 
 	stsToken := tokens[serviceAccountTokenAudienceSTS]
 	if stsToken == nil {
-		klog.Errorf("credentialprovider: `authenticationSource` configured to `pod` but no service account tokens for %s received. Please make sure to enable `podInfoOnMountCompat`, see "+podLevelCredentialsDocsPage, serviceAccountTokenAudienceSTS)
+		klog.Errorf("credentialprovider: `authenticationSource` configured to `pod` but no service account token for %s received. Please make sure to enable `podInfoOnMountCompat`, see "+podLevelCredentialsDocsPage, serviceAccountTokenAudienceSTS)
 		return nil, status.Errorf(codes.InvalidArgument, "Missing service account token for %s", serviceAccountTokenAudienceSTS)
 	}
 
-	roleARN, err := c.findPodServiceAccountRole(ctx, provideCtx)
-	if err != nil {
-		return nil, err
-	}
-
-	region, err := c.stsRegion(provideCtx)
-	if err != nil {
-		return nil, status.Errorf(codes.InvalidArgument, "Failed to detect STS AWS Region, please explicitly set the AWS Region, see "+stsConfigDocsPage)
-	}
-
-	defaultRegion := os.Getenv(envprovider.EnvDefaultRegion)
-	if defaultRegion == "" {
-		defaultRegion = region
-	}
-
-	podID := provideCtx.GetCredentialPodID()
-	volumeID := provideCtx.VolumeID
-	if podID == "" {
-		return nil, status.Error(codes.InvalidArgument, "Missing Pod info. Please make sure to enable `podInfoOnMountCompat`, see "+podLevelCredentialsDocsPage)
-	}
-
-	tokenName := podLevelServiceAccountTokenName(podID, volumeID)
-
-	err = renameio.WriteFile(filepath.Join(provideCtx.WritePath, tokenName), []byte(stsToken.Token), CredentialFilePerm)
-	if err != nil {
-		return nil, status.Errorf(codes.Internal, "Failed to write service account token: %v", err)
+	eksToken := tokens[serviceAccountTokenAudiencePodIdentity]
+	if util.UsePodMounter() && eksToken == nil {
+		klog.Errorf("credentialprovider: `authenticationSource` configured to `pod` but no service account token for %s received. Please make sure to enable `podInfoOnMountCompat`, see "+podLevelCredentialsDocsPage, serviceAccountTokenAudiencePodIdentity)
+		return nil, status.Errorf(codes.InvalidArgument, "Missing service account token for %s", serviceAccountTokenAudiencePodIdentity)
 	}
 
+	// 2. Create environment to be returned with common variables (used in both cases: IRSA and EKS PI)
 	podNamespace := provideCtx.PodNamespace
 	podServiceAccount := provideCtx.ServiceAccountName
 	cacheKey := podNamespace + "/" + podServiceAccount
 
-	return envprovider.Environment{
-		envprovider.EnvRoleARN:              roleARN,
-		envprovider.EnvWebIdentityTokenFile: filepath.Join(provideCtx.EnvPath, tokenName),
-
-		envprovider.EnvRegion:        region,
-		envprovider.EnvDefaultRegion: defaultRegion,
-
+	env := envprovider.Environment{
 		envprovider.EnvEC2MetadataDisabled: "true",
 
 		// TODO: These were needed with `systemd` but probably won't be necessary with containerization.
 		envprovider.EnvMountpointCacheKey:    cacheKey,
 		envprovider.EnvConfigFile:            filepath.Join(provideCtx.EnvPath, "disable-config"),
 		envprovider.EnvSharedCredentialsFile: filepath.Join(provideCtx.EnvPath, "disable-credentials"),
-	}, nil
+	}
+
+	// 3. Provide credentials with IRSA. If not configured, provide credentials with EKS Pod Identity instead.
+	irsaCredentialsEnvironment, irsaCredentialsEnvironmentError := c.createIRSACredentialsEnvironment(ctx, provideCtx)
+	if irsaCredentialsEnvironmentError == nil {
+		klog.V(4).Infof("Providing credentials from pod with STS Web Identity provider (IRSA)")
+
+		// Copy STS Token file to WritePath
+		tokenName := podLevelSTSWebIdentityServiceAccountTokenName(podID, provideCtx.VolumeID)
+		err := renameio.WriteFile(filepath.Join(provideCtx.WritePath, tokenName), []byte(stsToken.Token), CredentialFilePerm)
+		if err != nil {
+			return nil, status.Errorf(codes.Internal, "Failed to write service account STS token: %v", err)
+		}
+
+		env.Merge(irsaCredentialsEnvironment)
+		return env, nil
+	} else if errors.Is(irsaCredentialsEnvironmentError, errMissingServiceAccountAnnotationForIRSA) {
+		if !util.UsePodMounter() {
+			return nil, status.Errorf(codes.InvalidArgument, "Missing role annotation on pod's service account %s/%s", podNamespace, podServiceAccount)
+		}
+
+		klog.V(4).Infof("Providing credentials from pod with Container credential provider (EKS Pod Identity)")
+		eksPodIdentityCredentialsEnvironment := c.createEKSPodIdentityCredentialsEnvironment(provideCtx)
+
+		// Copy EKS Token file to WritePath
+		tokenNameEKS := podLevelEksPodIdentityServiceAccountTokenName(podID, provideCtx.VolumeID)
+		err := renameio.WriteFile(filepath.Join(provideCtx.WritePath, tokenNameEKS), []byte(eksToken.Token), CredentialFilePerm)
+		if err != nil {
+			return nil, status.Errorf(codes.Internal, "Failed to write service account EKS Pod Identity token: %v", err)
+		}
+
+		env.Merge(eksPodIdentityCredentialsEnvironment)
+		return env, nil
+	}
+
+	klog.V(4).Infof("Error providing credentials from pod with STS Web Identity provider (IRSA)")
+	return nil, irsaCredentialsEnvironmentError
 }
 
 // cleanupFromPod removes any credential files that were created for pod-level authentication authentication via [Provider.provideFromPod].
 func (c *Provider) cleanupFromPod(cleanupCtx CleanupContext) error {
-	tokenName := podLevelServiceAccountTokenName(cleanupCtx.PodID, cleanupCtx.VolumeID)
-	tokenPath := filepath.Join(cleanupCtx.WritePath, tokenName)
-	err := os.Remove(tokenPath)
-	if err != nil && errors.Is(err, fs.ErrNotExist) {
-		return nil
+	cleanupToken := func(tokenName string) error {
+		tokenPath := filepath.Join(cleanupCtx.WritePath, tokenName)
+		err := os.Remove(tokenPath)
+		if err != nil && errors.Is(err, fs.ErrNotExist) {
+			return nil
+		}
+
+		return err
 	}
-	return err
+
+	tokenNameSTS := podLevelSTSWebIdentityServiceAccountTokenName(cleanupCtx.PodID, cleanupCtx.VolumeID)
+	err := cleanupToken(tokenNameSTS)
+	if err != nil {
+		return status.Errorf(codes.Internal, "Failed to cleanup service account STS token: %v", err)
+	}
+
+	tokenNameEKS := podLevelEksPodIdentityServiceAccountTokenName(cleanupCtx.PodID, cleanupCtx.VolumeID)
+	err = cleanupToken(tokenNameEKS)
+	if err != nil {
+		status.Errorf(codes.Internal, "Failed to cleanup service account EKS Pod Identity token: %v", err)
+	}
+
+	return nil
 }
 
+var errMissingServiceAccountAnnotationForIRSA = errors.New("Missing role annotation on pod's service account")
+
 // findPodServiceAccountRole tries to provide associated AWS IAM role for service account specified in the volume context.
 func (c *Provider) findPodServiceAccountRole(ctx context.Context, provideCtx ProvideContext) (string, error) {
+	podNamespace := provideCtx.PodNamespace
+	podServiceAccount := provideCtx.ServiceAccountName
+
 	// In PodMounter we get IAM Role ARN from MountpointS3PodAttachment custom resource
-	if provideCtx.ServiceAccountEKSRoleARN != "" {
-		return provideCtx.ServiceAccountEKSRoleARN, nil
+	if util.UsePodMounter() {
+		if provideCtx.ServiceAccountEKSRoleARN != "" {
+			return provideCtx.ServiceAccountEKSRoleARN, nil
+		} else {
+			return "", errMissingServiceAccountAnnotationForIRSA
+		}
 	}
 
-	podNamespace := provideCtx.PodNamespace
-	podServiceAccount := provideCtx.ServiceAccountName
 	if podNamespace == "" || podServiceAccount == "" {
 		klog.Error("credentialprovider: `authenticationSource` configured to `pod` but no pod info found. Please make sure to enable `podInfoOnMountCompat`, see " + podLevelCredentialsDocsPage)
 		return "", status.Error(codes.InvalidArgument, "Missing Pod info. Please make sure to enable `podInfoOnMountCompat`, see "+podLevelCredentialsDocsPage)
@@ -133,15 +177,63 @@ func (c *Provider) findPodServiceAccountRole(ctx context.Context, provideCtx Pro
 	roleArn := response.Annotations[serviceAccountRoleAnnotation]
 	if roleArn == "" {
 		klog.Error("credentialprovider: `authenticationSource` configured to `pod` but pod's service account is not annotated with a role, see " + podLevelCredentialsDocsPage)
-		return "", status.Errorf(codes.InvalidArgument, "Missing role annotation on pod's service account %s/%s", podNamespace, podServiceAccount)
+		return "", errMissingServiceAccountAnnotationForIRSA
 	}
 
 	return roleArn, nil
 }
 
-// podLevelServiceAccountTokenName returns service account token name for Pod-level identity.
+// podLevelSTSWebIdentityServiceAccountTokenName returns service account token name for Pod-level identity.
 // It escapes from slashes to make this token name path-safe.
-func podLevelServiceAccountTokenName(podID string, volumeID string) string {
+func podLevelSTSWebIdentityServiceAccountTokenName(podID string, volumeID string) string {
 	id := escapedVolumeIdentifier(podID, volumeID)
 	return id + ".token"
 }
+
+// podLevelEksPodIdentityServiceAccountTokenName returns service account token name for Pod-level identity with EKS Pod Identity.
+// It escapes from slashes to make this token name path-safe.
+func podLevelEksPodIdentityServiceAccountTokenName(podID string, volumeID string) string {
+	id := escapedVolumeIdentifier(podID, volumeID)
+	return id + "-eks-pod-identity.token"
+}
+
+// createIRSACredentialsEnvironment creates an environment with the environment variables needed for pod-level authentication with IRSA
+func (c *Provider) createIRSACredentialsEnvironment(ctx context.Context, provideCtx ProvideContext) (envprovider.Environment, error) {
+	roleARN, err := c.findPodServiceAccountRole(ctx, provideCtx)
+	if err != nil {
+		return nil, err
+	}
+
+	region, err := c.stsRegion(provideCtx)
+	if err != nil {
+		return nil, status.Errorf(codes.InvalidArgument, "Failed to detect STS AWS Region, please explicitly set the AWS Region, see "+stsConfigDocsPage)
+	}
+
+	defaultRegion := os.Getenv(envprovider.EnvDefaultRegion)
+	if defaultRegion == "" {
+		defaultRegion = region
+	}
+
+	podID := provideCtx.GetCredentialPodID()
+	tokenName := podLevelSTSWebIdentityServiceAccountTokenName(podID, provideCtx.VolumeID)
+	tokenFile := filepath.Join(provideCtx.EnvPath, tokenName)
+
+	return envprovider.Environment{
+		envprovider.EnvRoleARN:              roleARN,
+		envprovider.EnvWebIdentityTokenFile: tokenFile,
+		envprovider.EnvRegion:               region,
+		envprovider.EnvDefaultRegion:        defaultRegion,
+	}, nil
+}
+
+// createEKSPodIdentityCredentialsEnvironment creates an environment with the environment variables needed for pod-level authentication with EKS Pod Identity
+func (c *Provider) createEKSPodIdentityCredentialsEnvironment(provideCtx ProvideContext) envprovider.Environment {
+	podID := provideCtx.GetCredentialPodID()
+	tokenName := podLevelEksPodIdentityServiceAccountTokenName(podID, provideCtx.VolumeID)
+	tokenFile := filepath.Join(provideCtx.EnvPath, tokenName)
+
+	return envprovider.Environment{
+		envprovider.EnvContainerCredentialsFullURI:     podIdentityCredURI,
+		envprovider.EnvContainerAuthorizationTokenFile: tokenFile,
+	}
+}