EKS Pod Identity support for driver level credentials (#451)

This PR is a follow-up to
#449 because we
need the E2E tests to run with the newly added `cluster-name` parameter.
This is not possible in a PR created from a fork.

*Description of changes:* With this change, we will support the
configuration of driver level credentials with EKS Pod Identity. In EKS
clusters, this is an alternative to IRSA with an easier configuration
process.

*Key Changes:*

- Credential Provider
- In `provider_driver.go` the container credentials used for EKS Pod
Identity are fetched from the driver pod and forwarded to the Mountpoint
process. This is done similarly to STS Web Identity credentials used for
IRSA.
- Added unit tests covering different credential setup scenarios
(including combinations)

- Testing
  - Added cluster name as parameter for E2E tests
- Added E2e tests covering EKS Pod Identity setup scenarios with IAM
roles of varying levels of S3 access
  - Manual testing performed on personal EKS cluster

By submitting this pull request, I confirm that you can use, modify,
copy, and redistribute this contribution, under the terms of your
choice.

---------

Signed-off-by: Renan Magagnin <renanmag@amazon.co.uk>

Author: renanmagagnin

pkg/driver/node/credentialprovider/provider_driver.go

@@ -13,7 +13,8 @@ import (
 )
 
 const (
-	driverLevelServiceAccountTokenName = "token"
+	webIdentityServiceAccountTokenName    = "token"
+	eksPodIdentityServiceAccountTokenName = "eks-pod-identity-token"
 )
 
 // provideFromDriver provides driver-level AWS credentials.
@@ -26,6 +27,7 @@ func (c *Provider) provideFromDriver(provideCtx ProvideContext) (envprovider.Env
 	accessKeyID := os.Getenv(envprovider.EnvAccessKeyID)
 	secretAccessKey := os.Getenv(envprovider.EnvSecretAccessKey)
 	if accessKeyID != "" && secretAccessKey != "" {
+		klog.V(4).Infof("Providing credentials from driver with Long-term AWS credentials")
 		sessionToken := os.Getenv(envprovider.EnvSessionToken)
 		longTermCredsEnv, err := provideLongTermCredentialsFromDriver(provideCtx, accessKeyID, secretAccessKey, sessionToken)
 		if err != nil {
@@ -37,6 +39,7 @@ func (c *Provider) provideFromDriver(provideCtx ProvideContext) (envprovider.Env
 	} else {
 		// Profile provider
 		// TODO: This is not officially supported and won't work by default with containerization.
+		klog.V(4).Infof("Providing credentials from driver with Profile provider")
 		configFile := os.Getenv(envprovider.EnvConfigFile)
 		sharedCredentialsFile := os.Getenv(envprovider.EnvSharedCredentialsFile)
 		if configFile != "" && sharedCredentialsFile != "" {
@@ -45,10 +48,11 @@ func (c *Provider) provideFromDriver(provideCtx ProvideContext) (envprovider.Env
 		}
 	}
 
-	// STS Web Identity provider
+	// STS Web Identity provider (IRSA)
 	webIdentityTokenFile := os.Getenv(envprovider.EnvWebIdentityTokenFile)
 	roleARN := os.Getenv(envprovider.EnvRoleARN)
 	if webIdentityTokenFile != "" && roleARN != "" {
+		klog.V(4).Infof("Providing credentials from driver with STS Web Identity provider (IRSA)")
 		stsWebIdentityCredsEnv, err := provideStsWebIdentityCredentialsFromDriver(provideCtx)
 		if err != nil {
 			klog.V(4).ErrorS(err, "credentialprovider: Failed to provide STS Web Identity credentials from driver")
@@ -58,6 +62,19 @@ func (c *Provider) provideFromDriver(provideCtx ProvideContext) (envprovider.Env
 		env.Merge(stsWebIdentityCredsEnv)
 	}
 
+	// Container credential provider (EKS Pod Identity)
+	containerAuthorizationTokenFile := os.Getenv(envprovider.EnvContainerAuthorizationTokenFile)
+	containerCredentialsFullURI := os.Getenv(envprovider.EnvContainerCredentialsFullURI)
+	if util.UsePodMounter() && containerAuthorizationTokenFile != "" && containerCredentialsFullURI != "" {
+		klog.V(4).Infof("Providing credentials from driver with Container credential provider (EKS Pod Identity)")
+		containerCredsEnv, err := provideContainerCredentialsFromDriver(provideCtx, containerAuthorizationTokenFile, containerCredentialsFullURI)
+		if err != nil {
+			klog.V(4).ErrorS(err, "credentialprovider: Failed to provide container credentials from driver")
+			return nil, err
+		}
+		env.Merge(containerCredsEnv)
+	}
+
 	return env, nil
 }
 
@@ -74,15 +91,30 @@ func (c *Provider) cleanupFromDriver(cleanupCtx CleanupContext) error {
 // It basically copies driver's injected service account token to [provideCtx.WritePath].
 func provideStsWebIdentityCredentialsFromDriver(provideCtx ProvideContext) (envprovider.Environment, error) {
 	driverServiceAccountTokenFile := os.Getenv(envprovider.EnvWebIdentityTokenFile)
-	tokenFile := filepath.Join(provideCtx.WritePath, driverLevelServiceAccountTokenName)
+	tokenFile := filepath.Join(provideCtx.WritePath, webIdentityServiceAccountTokenName)
 	err := util.ReplaceFile(tokenFile, driverServiceAccountTokenFile, CredentialFilePerm)
 	if err != nil {
 		return nil, fmt.Errorf("credentialprovider: sts-web-identity: failed to copy driver's service account token: %w", err)
 	}
 
 	return envprovider.Environment{
 		envprovider.EnvRoleARN:              os.Getenv(envprovider.EnvRoleARN),
-		envprovider.EnvWebIdentityTokenFile: filepath.Join(provideCtx.EnvPath, driverLevelServiceAccountTokenName),
+		envprovider.EnvWebIdentityTokenFile: filepath.Join(provideCtx.EnvPath, webIdentityServiceAccountTokenName),
+	}, nil
+}
+
+// provideContainerCredentialsFromDriver provides Container credentials from the driver's service account.
+// It basically copies driver's injected service account token to [provideCtx.WritePath].
+func provideContainerCredentialsFromDriver(provideCtx ProvideContext, containerAuthorizationTokenFile string, containerCredentialsFullURI string) (envprovider.Environment, error) {
+	tokenFile := filepath.Join(provideCtx.WritePath, eksPodIdentityServiceAccountTokenName)
+	err := util.ReplaceFile(tokenFile, containerAuthorizationTokenFile, CredentialFilePerm)
+	if err != nil {
+		return nil, fmt.Errorf("credentialprovider: container: failed to copy driver's service account token: %w", err)
+	}
+
+	return envprovider.Environment{
+		envprovider.EnvContainerAuthorizationTokenFile: filepath.Join(provideCtx.EnvPath, eksPodIdentityServiceAccountTokenName),
+		envprovider.EnvContainerCredentialsFullURI:     containerCredentialsFullURI,
 	}, nil
 }
 

pkg/driver/node/credentialprovider/provider_test.go

@@ -28,12 +28,16 @@ const testSessionToken = "test-session-token"
 const testRoleARN = "arn:aws:iam::111122223333:role/pod-a-role"
 const testWebIdentityToken = "test-web-identity-token"
 
+const testContainerAuthorizationToken = "test-container-authorization-token"
+const testContainerCredentialsFullURI = "http://169.254.170.23/v1/credentials"
+
 const testPodID = "2a17db00-0bf3-4052-9b3f-6c89dcee5d79"
 const testVolumeID = "test-vol"
 const testProfilePrefix = testPodID + "-" + testVolumeID + "-"
 
 const testPodLevelServiceAccountToken = testPodID + "-" + testVolumeID + ".token"
-const testDriverLevelServiceAccountToken = "token"
+const testWebIdentityServiceAccountToken = "token"
+const testEKSPodIdentityServiceAccountToken = "eks-pod-identity-token"
 
 const testPodServiceAccount = "test-sa"
 const testPodNamespace = "test-ns"
@@ -98,9 +102,27 @@ func TestProvidingDriverLevelCredentials(t *testing.T) {
 			assert.Equals(t, credentialprovider.AuthenticationSourceDriver, source)
 			assert.Equals(t, envprovider.Environment{
 				"AWS_ROLE_ARN":                testRoleARN,
-				"AWS_WEB_IDENTITY_TOKEN_FILE": filepath.Join(testEnvPath, testDriverLevelServiceAccountToken),
+				"AWS_WEB_IDENTITY_TOKEN_FILE": filepath.Join(testEnvPath, testWebIdentityServiceAccountToken),
+			}, env)
+			assertWebIdentityTokenFile(t, filepath.Join(writePath, testWebIdentityServiceAccountToken))
+		}
+	})
+
+	t.Run("only container credentials", func(t *testing.T) {
+		for _, authSource := range authenticationSourceVariants {
+			setEnvForContainerCredentials(t)
+
+			writePath := t.TempDir()
+			provideCtx := provideCtx(t, writePath, authSource)
+
+			env, source, err := provider.Provide(context.Background(), provideCtx)
+			assert.NoError(t, err)
+			assert.Equals(t, credentialprovider.AuthenticationSourceDriver, source)
+			assert.Equals(t, envprovider.Environment{
+				"AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE": filepath.Join(testEnvPath, testEKSPodIdentityServiceAccountToken),
+				"AWS_CONTAINER_CREDENTIALS_FULL_URI":     testContainerCredentialsFullURI,
 			}, env)
-			assertWebIdentityTokenFile(t, filepath.Join(writePath, testDriverLevelServiceAccountToken))
+			assertContainerTokenFile(t, filepath.Join(writePath, testEKSPodIdentityServiceAccountToken))
 		}
 	})
 
@@ -148,10 +170,55 @@ func TestProvidingDriverLevelCredentials(t *testing.T) {
 				"AWS_CONFIG_FILE":             "/test-env/" + testProfilePrefix + "s3-csi-config",
 				"AWS_SHARED_CREDENTIALS_FILE": "/test-env/" + testProfilePrefix + "s3-csi-credentials",
 				"AWS_ROLE_ARN":                testRoleARN,
-				"AWS_WEB_IDENTITY_TOKEN_FILE": filepath.Join(testEnvPath, testDriverLevelServiceAccountToken),
+				"AWS_WEB_IDENTITY_TOKEN_FILE": filepath.Join(testEnvPath, testWebIdentityServiceAccountToken),
+			}, env)
+			assertLongTermCredentials(t, writePath)
+			assertWebIdentityTokenFile(t, filepath.Join(writePath, testWebIdentityServiceAccountToken))
+		}
+	})
+
+	t.Run("long-term and container credentials", func(t *testing.T) {
+		for _, authSource := range authenticationSourceVariants {
+			setEnvForLongTermCredentials(t)
+			setEnvForContainerCredentials(t)
+
+			writePath := t.TempDir()
+			provideCtx := provideCtx(t, writePath, authSource)
+
+			env, source, err := provider.Provide(context.Background(), provideCtx)
+			assert.NoError(t, err)
+			assert.Equals(t, credentialprovider.AuthenticationSourceDriver, source)
+			assert.Equals(t, envprovider.Environment{
+				"AWS_PROFILE":                            testProfilePrefix + "s3-csi",
+				"AWS_CONFIG_FILE":                        "/test-env/" + testProfilePrefix + "s3-csi-config",
+				"AWS_SHARED_CREDENTIALS_FILE":            "/test-env/" + testProfilePrefix + "s3-csi-credentials",
+				"AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE": filepath.Join(testEnvPath, testEKSPodIdentityServiceAccountToken),
+				"AWS_CONTAINER_CREDENTIALS_FULL_URI":     testContainerCredentialsFullURI,
 			}, env)
 			assertLongTermCredentials(t, writePath)
-			assertWebIdentityTokenFile(t, filepath.Join(writePath, testDriverLevelServiceAccountToken))
+			assertContainerTokenFile(t, filepath.Join(writePath, testEKSPodIdentityServiceAccountToken))
+		}
+	})
+
+	t.Run("sts web identity credentials and containter credentials", func(t *testing.T) {
+		for _, authSource := range authenticationSourceVariants {
+			setEnvForContainerCredentials(t)
+			setEnvForStsWebIdentityCredentials(t)
+
+			writePath := t.TempDir()
+			provideCtx := provideCtx(t, writePath, authSource)
+
+			env, source, err := provider.Provide(context.Background(), provideCtx)
+			assert.NoError(t, err)
+			assert.Equals(t, credentialprovider.AuthenticationSourceDriver, source)
+			assert.Equals(t, envprovider.Environment{
+				"AWS_ROLE_ARN":                           testRoleARN,
+				"AWS_WEB_IDENTITY_TOKEN_FILE":            filepath.Join(testEnvPath, testWebIdentityServiceAccountToken),
+				"AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE": filepath.Join(testEnvPath, testEKSPodIdentityServiceAccountToken),
+				"AWS_CONTAINER_CREDENTIALS_FULL_URI":     testContainerCredentialsFullURI,
+			}, env)
+			assertContainerTokenFile(t, filepath.Join(writePath, testEKSPodIdentityServiceAccountToken))
+			assertWebIdentityTokenFile(t, filepath.Join(writePath, testWebIdentityServiceAccountToken))
 		}
 	})
 
@@ -215,6 +282,31 @@ func TestProvidingDriverLevelCredentials(t *testing.T) {
 		assert.Equals(t, envprovider.Environment{}, env)
 	})
 
+	t.Run("incomplete container credentials", func(t *testing.T) {
+		// Only set container credentials full URI without token file
+		t.Setenv("AWS_CONTAINER_CREDENTIALS_FULL_URI", testContainerCredentialsFullURI)
+
+		provider := credentialprovider.New(nil, dummyRegionProvider)
+
+		provideCtx := provideCtx(t, t.TempDir(), credentialprovider.AuthenticationSourceDriver)
+
+		env, source, err := provider.Provide(context.Background(), provideCtx)
+		assert.NoError(t, err)
+		assert.Equals(t, credentialprovider.AuthenticationSourceDriver, source)
+		assert.Equals(t, envprovider.Environment{}, env)
+
+		// Only set token file without role ARN
+		tokenPath := filepath.Join(t.TempDir(), "token")
+		assert.NoError(t, os.WriteFile(tokenPath, []byte(testContainerAuthorizationToken), 0600))
+		t.Setenv("AWS_ROLE_ARN", "")
+		t.Setenv("AWS_WEB_IDENTITY_TOKEN_FILE", tokenPath)
+
+		env, source, err = provider.Provide(context.Background(), provideCtx)
+		assert.NoError(t, err)
+		assert.Equals(t, credentialprovider.AuthenticationSourceDriver, source)
+		assert.Equals(t, envprovider.Environment{}, env)
+	})
+
 	t.Run("no credentials", func(t *testing.T) {
 		for _, authSource := range authenticationSourceVariants {
 			writePath := t.TempDir()
@@ -817,6 +909,16 @@ func TestCleanup(t *testing.T) {
 
 //-- Utilities for tests
 
+func provideCtx(t *testing.T, writePath string, authSource string) credentialprovider.ProvideContext {
+	return credentialprovider.ProvideContext{
+		AuthenticationSource: authSource,
+		WritePath:            writePath,
+		EnvPath:              testEnvPath,
+		PodID:                testPodID,
+		VolumeID:             testVolumeID,
+	}
+}
+
 func setEnvForLongTermCredentials(t *testing.T) {
 	t.Setenv("AWS_ACCESS_KEY_ID", testAccessKeyID)
 	t.Setenv("AWS_SECRET_ACCESS_KEY", testSecretAccessKey)
@@ -848,6 +950,18 @@ func setEnvForStsWebIdentityCredentials(t *testing.T) {
 	t.Setenv("AWS_WEB_IDENTITY_TOKEN_FILE", tokenPath)
 }
 
+func setEnvForContainerCredentials(t *testing.T) {
+	t.Helper()
+
+	t.Setenv("MOUNTER_KIND", "pod")
+
+	tokenPath := filepath.Join(t.TempDir(), "token")
+	assert.NoError(t, os.WriteFile(tokenPath, []byte(testContainerAuthorizationToken), 0600))
+
+	t.Setenv("AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE", tokenPath)
+	t.Setenv("AWS_CONTAINER_CREDENTIALS_FULL_URI", testContainerCredentialsFullURI)
+}
+
 func assertWebIdentityTokenFile(t *testing.T, path string) {
 	t.Helper()
 
@@ -856,6 +970,14 @@ func assertWebIdentityTokenFile(t *testing.T, path string) {
 	assert.Equals(t, []byte(testWebIdentityToken), got)
 }
 
+func assertContainerTokenFile(t *testing.T, path string) {
+	t.Helper()
+
+	got, err := os.ReadFile(path)
+	assert.NoError(t, err)
+	assert.Equals(t, []byte(testContainerAuthorizationToken), got)
+}
+
 type tokens = map[string]struct {
 	Token               string `json:"token"`
 	ExpirationTimestamp time.Time

pkg/driver/node/envprovider/provider.go

@@ -9,20 +9,22 @@ import (
 )
 
 const (
-	EnvRegion                = "AWS_REGION"
-	EnvDefaultRegion         = "AWS_DEFAULT_REGION"
-	EnvSTSRegionalEndpoints  = "AWS_STS_REGIONAL_ENDPOINTS"
-	EnvMaxAttempts           = "AWS_MAX_ATTEMPTS"
-	EnvProfile               = "AWS_PROFILE"
-	EnvConfigFile            = "AWS_CONFIG_FILE"
-	EnvSharedCredentialsFile = "AWS_SHARED_CREDENTIALS_FILE"
-	EnvRoleARN               = "AWS_ROLE_ARN"
-	EnvWebIdentityTokenFile  = "AWS_WEB_IDENTITY_TOKEN_FILE"
-	EnvEC2MetadataDisabled   = "AWS_EC2_METADATA_DISABLED"
-	EnvAccessKeyID           = "AWS_ACCESS_KEY_ID"
-	EnvSecretAccessKey       = "AWS_SECRET_ACCESS_KEY"
-	EnvSessionToken          = "AWS_SESSION_TOKEN"
-	EnvMountpointCacheKey    = "UNSTABLE_MOUNTPOINT_CACHE_KEY"
+	EnvRegion                          = "AWS_REGION"
+	EnvDefaultRegion                   = "AWS_DEFAULT_REGION"
+	EnvSTSRegionalEndpoints            = "AWS_STS_REGIONAL_ENDPOINTS"
+	EnvMaxAttempts                     = "AWS_MAX_ATTEMPTS"
+	EnvProfile                         = "AWS_PROFILE"
+	EnvConfigFile                      = "AWS_CONFIG_FILE"
+	EnvSharedCredentialsFile           = "AWS_SHARED_CREDENTIALS_FILE"
+	EnvRoleARN                         = "AWS_ROLE_ARN"
+	EnvWebIdentityTokenFile            = "AWS_WEB_IDENTITY_TOKEN_FILE"
+	EnvContainerAuthorizationTokenFile = "AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE"
+	EnvContainerCredentialsFullURI     = "AWS_CONTAINER_CREDENTIALS_FULL_URI"
+	EnvEC2MetadataDisabled             = "AWS_EC2_METADATA_DISABLED"
+	EnvAccessKeyID                     = "AWS_ACCESS_KEY_ID"
+	EnvSecretAccessKey                 = "AWS_SECRET_ACCESS_KEY"
+	EnvSessionToken                    = "AWS_SESSION_TOKEN"
+	EnvMountpointCacheKey              = "UNSTABLE_MOUNTPOINT_CACHE_KEY"
 )
 
 // Key represents an environment variable name.

tests/e2e-kubernetes/e2e_test.go

@@ -23,6 +23,7 @@ func init() {
 
 	flag.StringVar(&CommitId, "commit-id", "local", "commit id will be used to name buckets")
 	flag.StringVar(&BucketRegion, "bucket-region", "us-east-1", "region where temporary buckets will be created")
+	flag.StringVar(&ClusterName, "cluster-name", "", "name of the cluster")
 	flag.StringVar(&BucketPrefix, "bucket-prefix", "local", "prefix for temporary buckets")
 	flag.BoolVar(&Performance, "performance", false, "run performance tests")
 	flag.BoolVar(&IMDSAvailable, "imds-available", false, "indicates whether instance metadata service is available")
@@ -31,6 +32,7 @@ func init() {
 
 	s3client.DefaultRegion = BucketRegion
 	custom_testsuites.DefaultRegion = BucketRegion
+	custom_testsuites.ClusterName = ClusterName
 	custom_testsuites.IMDSAvailable = IMDSAvailable
 	custom_testsuites.IsPodMounter = IsPodMounter
 }

tests/e2e-kubernetes/go.mod

@@ -3,7 +3,7 @@ module github.com/awslabs/aws-s3-csi-driver/tests/e2e-kubernetes
 go 1.24
 
 require (
-	github.com/aws/aws-sdk-go-v2 v1.30.5
+	github.com/aws/aws-sdk-go-v2 v1.36.3
 	github.com/aws/aws-sdk-go-v2/config v1.27.33
 	github.com/aws/aws-sdk-go-v2/service/iam v1.34.3
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.47.3
@@ -29,17 +29,19 @@ require (
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.3 // indirect
 	github.com/aws/aws-sdk-go-v2/credentials v1.17.32 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.13 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.17 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.17 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.8 // indirect
+	github.com/aws/aws-sdk-go-v2/service/eks v1.64.0
+	github.com/aws/aws-sdk-go-v2/service/eksauth v1.8.2
 	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.4 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.8 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.19 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.8 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sso v1.22.7 // indirect
 	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.7 // indirect
-	github.com/aws/smithy-go v1.20.4 // indirect
+	github.com/aws/smithy-go v1.22.2 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/blang/semver/v4 v4.0.0 // indirect
 	github.com/cenkalti/backoff/v4 v4.3.0 // indirect