EKS Pod Identity support for driver level credentials

pkg/driver/node/credentialprovider/provider_driver.go

@@ -13,7 +13,8 @@ import (
 )
 
 const (
-	driverLevelServiceAccountTokenName = "token"
+	webIdentityServiceAccountTokenName    = "token"
+	eksPodIdentityServiceAccountTokenName = "eks-pod-identity-token"
 )
 
 // provideFromDriver provides driver-level AWS credentials.
@@ -26,6 +27,7 @@ func (c *Provider) provideFromDriver(provideCtx ProvideContext) (envprovider.Env
 	accessKeyID := os.Getenv(envprovider.EnvAccessKeyID)
 	secretAccessKey := os.Getenv(envprovider.EnvSecretAccessKey)
 	if accessKeyID != "" && secretAccessKey != "" {
+		klog.V(4).Infof("Providing credentials from driver with Long-term AWS credentials")
 		sessionToken := os.Getenv(envprovider.EnvSessionToken)
 		longTermCredsEnv, err := provideLongTermCredentialsFromDriver(provideCtx, accessKeyID, secretAccessKey, sessionToken)
 		if err != nil {
@@ -37,6 +39,7 @@ func (c *Provider) provideFromDriver(provideCtx ProvideContext) (envprovider.Env
 	} else {
 		// Profile provider
 		// TODO: This is not officially supported and won't work by default with containerization.
+		klog.V(4).Infof("Providing credentials from driver with Profile provider")
 		configFile := os.Getenv(envprovider.EnvConfigFile)
 		sharedCredentialsFile := os.Getenv(envprovider.EnvSharedCredentialsFile)
 		if configFile != "" && sharedCredentialsFile != "" {
@@ -45,10 +48,11 @@ func (c *Provider) provideFromDriver(provideCtx ProvideContext) (envprovider.Env
 		}
 	}
 
-	// STS Web Identity provider
+	// STS Web Identity provider (IRSA)
 	webIdentityTokenFile := os.Getenv(envprovider.EnvWebIdentityTokenFile)
 	roleARN := os.Getenv(envprovider.EnvRoleARN)
 	if webIdentityTokenFile != "" && roleARN != "" {
+		klog.V(4).Infof("Providing credentials from driver with STS Web Identity provider (IRSA)")
 		stsWebIdentityCredsEnv, err := provideStsWebIdentityCredentialsFromDriver(provideCtx)
 		if err != nil {
 			klog.V(4).ErrorS(err, "credentialprovider: Failed to provide STS Web Identity credentials from driver")
@@ -58,6 +62,19 @@ func (c *Provider) provideFromDriver(provideCtx ProvideContext) (envprovider.Env
 		env.Merge(stsWebIdentityCredsEnv)
 	}
 
+	// Container credential provider (EKS Pod Identity)
+	containerAuthorizationTokenFile := os.Getenv(envprovider.EnvContainerAuthorizationTokenFile)
+	containerCredentialsFullURI := os.Getenv(envprovider.EnvContainerCredentialsFullURI)
+	if util.UsePodMounter() && containerAuthorizationTokenFile != "" && containerCredentialsFullURI != "" {
+		klog.V(4).Infof("Providing credentials from driver with Container credential provider (EKS Pod Identity)")
+		containerCredsEnv, err := provideContainerCredentialsFromDriver(provideCtx, containerAuthorizationTokenFile, containerCredentialsFullURI)
+		if err != nil {
+			klog.V(4).ErrorS(err, "credentialprovider: Failed to provide container credentials from driver")
+			return nil, err
+		}
+		env.Merge(containerCredsEnv)
+	}
+
 	return env, nil
 }
 
@@ -74,15 +91,30 @@ func (c *Provider) cleanupFromDriver(cleanupCtx CleanupContext) error {
 // It basically copies driver's injected service account token to [provideCtx.WritePath].
 func provideStsWebIdentityCredentialsFromDriver(provideCtx ProvideContext) (envprovider.Environment, error) {
 	driverServiceAccountTokenFile := os.Getenv(envprovider.EnvWebIdentityTokenFile)
-	tokenFile := filepath.Join(provideCtx.WritePath, driverLevelServiceAccountTokenName)
+	tokenFile := filepath.Join(provideCtx.WritePath, webIdentityServiceAccountTokenName)
 	err := util.ReplaceFile(tokenFile, driverServiceAccountTokenFile, CredentialFilePerm)
 	if err != nil {
 		return nil, fmt.Errorf("credentialprovider: sts-web-identity: failed to copy driver's service account token: %w", err)
 	}
 
 	return envprovider.Environment{
 		envprovider.EnvRoleARN:              os.Getenv(envprovider.EnvRoleARN),
-		envprovider.EnvWebIdentityTokenFile: filepath.Join(provideCtx.EnvPath, driverLevelServiceAccountTokenName),
+		envprovider.EnvWebIdentityTokenFile: filepath.Join(provideCtx.EnvPath, webIdentityServiceAccountTokenName),
+	}, nil
+}
+
+// provideContainerCredentialsFromDriver provides Container credentials from the driver's service account.
+// It basically copies driver's injected service account token to [provideCtx.WritePath].
+func provideContainerCredentialsFromDriver(provideCtx ProvideContext, containerAuthorizationTokenFile string, containerCredentialsFullURI string) (envprovider.Environment, error) {
+	tokenFile := filepath.Join(provideCtx.WritePath, eksPodIdentityServiceAccountTokenName)
+	err := util.ReplaceFile(tokenFile, containerAuthorizationTokenFile, CredentialFilePerm)
+	if err != nil {
+		return nil, fmt.Errorf("credentialprovider: container: failed to copy driver's service account token: %w", err)
+	}
+
+	return envprovider.Environment{
+		envprovider.EnvContainerAuthorizationTokenFile: filepath.Join(provideCtx.EnvPath, eksPodIdentityServiceAccountTokenName),
+		envprovider.EnvContainerCredentialsFullURI:     containerCredentialsFullURI,
 	}, nil
 }
 

pkg/driver/node/credentialprovider/provider_test.go

@@ -28,12 +28,16 @@ const testSessionToken = "test-session-token"
 const testRoleARN = "arn:aws:iam::111122223333:role/pod-a-role"
 const testWebIdentityToken = "test-web-identity-token"
 
+const testContainerAuthorizationToken = "test-container-authorization-token"
+const testContainerCredentialsFullURI = "http://169.254.170.23/v1/credentials"
+
 const testPodID = "2a17db00-0bf3-4052-9b3f-6c89dcee5d79"
 const testVolumeID = "test-vol"
 const testProfilePrefix = testPodID + "-" + testVolumeID + "-"
 
 const testPodLevelServiceAccountToken = testPodID + "-" + testVolumeID + ".token"
-const testDriverLevelServiceAccountToken = "token"
+const testWebIdentityServiceAccountToken = "token"
+const testEKSPodIdentityServiceAccountToken = "eks-pod-identity-token"
 
 const testPodServiceAccount = "test-sa"
 const testPodNamespace = "test-ns"
@@ -98,9 +102,27 @@ func TestProvidingDriverLevelCredentials(t *testing.T) {
 			assert.Equals(t, credentialprovider.AuthenticationSourceDriver, source)
 			assert.Equals(t, envprovider.Environment{
 				"AWS_ROLE_ARN":                testRoleARN,
-				"AWS_WEB_IDENTITY_TOKEN_FILE": filepath.Join(testEnvPath, testDriverLevelServiceAccountToken),
+				"AWS_WEB_IDENTITY_TOKEN_FILE": filepath.Join(testEnvPath, testWebIdentityServiceAccountToken),
+			}, env)
+			assertWebIdentityTokenFile(t, filepath.Join(writePath, testWebIdentityServiceAccountToken))
+		}
+	})
+
+	t.Run("only container credentials", func(t *testing.T) {
+		for _, authSource := range authenticationSourceVariants {
+			setEnvForContainerCredentials(t)
+
+			writePath := t.TempDir()
+			provideCtx := provideCtx(t, writePath, authSource)
+
+			env, source, err := provider.Provide(context.Background(), provideCtx)
+			assert.NoError(t, err)
+			assert.Equals(t, credentialprovider.AuthenticationSourceDriver, source)
+			assert.Equals(t, envprovider.Environment{
+				"AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE": filepath.Join(testEnvPath, testEKSPodIdentityServiceAccountToken),
+				"AWS_CONTAINER_CREDENTIALS_FULL_URI":     testContainerCredentialsFullURI,
 			}, env)
-			assertWebIdentityTokenFile(t, filepath.Join(writePath, testDriverLevelServiceAccountToken))
+			assertContainerTokenFile(t, filepath.Join(writePath, testEKSPodIdentityServiceAccountToken))
 		}
 	})
 
@@ -148,10 +170,55 @@ func TestProvidingDriverLevelCredentials(t *testing.T) {
 				"AWS_CONFIG_FILE":             "/test-env/" + testProfilePrefix + "s3-csi-config",
 				"AWS_SHARED_CREDENTIALS_FILE": "/test-env/" + testProfilePrefix + "s3-csi-credentials",
 				"AWS_ROLE_ARN":                testRoleARN,
-				"AWS_WEB_IDENTITY_TOKEN_FILE": filepath.Join(testEnvPath, testDriverLevelServiceAccountToken),
+				"AWS_WEB_IDENTITY_TOKEN_FILE": filepath.Join(testEnvPath, testWebIdentityServiceAccountToken),
+			}, env)
+			assertLongTermCredentials(t, writePath)
+			assertWebIdentityTokenFile(t, filepath.Join(writePath, testWebIdentityServiceAccountToken))
+		}
+	})
+
+	t.Run("long-term and container credentials", func(t *testing.T) {
+		for _, authSource := range authenticationSourceVariants {
+			setEnvForLongTermCredentials(t)
+			setEnvForContainerCredentials(t)
+
+			writePath := t.TempDir()
+			provideCtx := provideCtx(t, writePath, authSource)
+
+			env, source, err := provider.Provide(context.Background(), provideCtx)
+			assert.NoError(t, err)
+			assert.Equals(t, credentialprovider.AuthenticationSourceDriver, source)
+			assert.Equals(t, envprovider.Environment{
+				"AWS_PROFILE":                            testProfilePrefix + "s3-csi",
+				"AWS_CONFIG_FILE":                        "/test-env/" + testProfilePrefix + "s3-csi-config",
+				"AWS_SHARED_CREDENTIALS_FILE":            "/test-env/" + testProfilePrefix + "s3-csi-credentials",
+				"AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE": filepath.Join(testEnvPath, testEKSPodIdentityServiceAccountToken),
+				"AWS_CONTAINER_CREDENTIALS_FULL_URI":     testContainerCredentialsFullURI,
 			}, env)
 			assertLongTermCredentials(t, writePath)
-			assertWebIdentityTokenFile(t, filepath.Join(writePath, testDriverLevelServiceAccountToken))
+			assertContainerTokenFile(t, filepath.Join(writePath, testEKSPodIdentityServiceAccountToken))
+		}
+	})
+
+	t.Run("sts web identity credentials and containter credentials", func(t *testing.T) {
+		for _, authSource := range authenticationSourceVariants {
+			setEnvForContainerCredentials(t)
+			setEnvForStsWebIdentityCredentials(t)
+
+			writePath := t.TempDir()
+			provideCtx := provideCtx(t, writePath, authSource)
+
+			env, source, err := provider.Provide(context.Background(), provideCtx)
+			assert.NoError(t, err)
+			assert.Equals(t, credentialprovider.AuthenticationSourceDriver, source)
+			assert.Equals(t, envprovider.Environment{
+				"AWS_ROLE_ARN":                           testRoleARN,
+				"AWS_WEB_IDENTITY_TOKEN_FILE":            filepath.Join(testEnvPath, testWebIdentityServiceAccountToken),
+				"AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE": filepath.Join(testEnvPath, testEKSPodIdentityServiceAccountToken),
+				"AWS_CONTAINER_CREDENTIALS_FULL_URI":     testContainerCredentialsFullURI,
+			}, env)
+			assertContainerTokenFile(t, filepath.Join(writePath, testEKSPodIdentityServiceAccountToken))
+			assertWebIdentityTokenFile(t, filepath.Join(writePath, testWebIdentityServiceAccountToken))
 		}
 	})
 
@@ -215,6 +282,31 @@ func TestProvidingDriverLevelCredentials(t *testing.T) {
 		assert.Equals(t, envprovider.Environment{}, env)
 	})
 
+	t.Run("incomplete container credentials", func(t *testing.T) {
+		// Only set container credentials full URI without token file
+		t.Setenv("AWS_CONTAINER_CREDENTIALS_FULL_URI", testContainerCredentialsFullURI)
+
+		provider := credentialprovider.New(nil, dummyRegionProvider)
+
+		provideCtx := provideCtx(t, t.TempDir(), credentialprovider.AuthenticationSourceDriver)
+
+		env, source, err := provider.Provide(context.Background(), provideCtx)
+		assert.NoError(t, err)
+		assert.Equals(t, credentialprovider.AuthenticationSourceDriver, source)
+		assert.Equals(t, envprovider.Environment{}, env)
+
+		// Only set token file without role ARN
+		tokenPath := filepath.Join(t.TempDir(), "token")
+		assert.NoError(t, os.WriteFile(tokenPath, []byte(testContainerAuthorizationToken), 0600))
+		t.Setenv("AWS_ROLE_ARN", "")
+		t.Setenv("AWS_WEB_IDENTITY_TOKEN_FILE", tokenPath)
+
+		env, source, err = provider.Provide(context.Background(), provideCtx)
+		assert.NoError(t, err)
+		assert.Equals(t, credentialprovider.AuthenticationSourceDriver, source)
+		assert.Equals(t, envprovider.Environment{}, env)
+	})
+
 	t.Run("no credentials", func(t *testing.T) {
 		for _, authSource := range authenticationSourceVariants {
 			writePath := t.TempDir()
@@ -817,6 +909,16 @@ func TestCleanup(t *testing.T) {
 
 //-- Utilities for tests
 
+func provideCtx(t *testing.T, writePath string, authSource string) credentialprovider.ProvideContext {
+	return credentialprovider.ProvideContext{
+		AuthenticationSource: authSource,
+		WritePath:            writePath,
+		EnvPath:              testEnvPath,
+		PodID:                testPodID,
+		VolumeID:             testVolumeID,
+	}
+}
+
 func setEnvForLongTermCredentials(t *testing.T) {
 	t.Setenv("AWS_ACCESS_KEY_ID", testAccessKeyID)
 	t.Setenv("AWS_SECRET_ACCESS_KEY", testSecretAccessKey)
@@ -848,6 +950,18 @@ func setEnvForStsWebIdentityCredentials(t *testing.T) {
 	t.Setenv("AWS_WEB_IDENTITY_TOKEN_FILE", tokenPath)
 }
 
+func setEnvForContainerCredentials(t *testing.T) {
+	t.Helper()
+
+	t.Setenv("MOUNTER_KIND", "pod")
+
+	tokenPath := filepath.Join(t.TempDir(), "token")
+	assert.NoError(t, os.WriteFile(tokenPath, []byte(testContainerAuthorizationToken), 0600))
+
+	t.Setenv("AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE", tokenPath)
+	t.Setenv("AWS_CONTAINER_CREDENTIALS_FULL_URI", testContainerCredentialsFullURI)
+}
+
 func assertWebIdentityTokenFile(t *testing.T, path string) {
 	t.Helper()
 
@@ -856,6 +970,14 @@ func assertWebIdentityTokenFile(t *testing.T, path string) {
 	assert.Equals(t, []byte(testWebIdentityToken), got)
 }
 
+func assertContainerTokenFile(t *testing.T, path string) {
+	t.Helper()
+
+	got, err := os.ReadFile(path)
+	assert.NoError(t, err)
+	assert.Equals(t, []byte(testContainerAuthorizationToken), got)
+}
+
 type tokens = map[string]struct {
 	Token               string `json:"token"`
 	ExpirationTimestamp time.Time

pkg/driver/node/envprovider/provider.go

@@ -9,20 +9,22 @@ import (
 )
 
 const (
-	EnvRegion                = "AWS_REGION"
-	EnvDefaultRegion         = "AWS_DEFAULT_REGION"
-	EnvSTSRegionalEndpoints  = "AWS_STS_REGIONAL_ENDPOINTS"
-	EnvMaxAttempts           = "AWS_MAX_ATTEMPTS"
-	EnvProfile               = "AWS_PROFILE"
-	EnvConfigFile            = "AWS_CONFIG_FILE"
-	EnvSharedCredentialsFile = "AWS_SHARED_CREDENTIALS_FILE"
-	EnvRoleARN               = "AWS_ROLE_ARN"
-	EnvWebIdentityTokenFile  = "AWS_WEB_IDENTITY_TOKEN_FILE"
-	EnvEC2MetadataDisabled   = "AWS_EC2_METADATA_DISABLED"
-	EnvAccessKeyID           = "AWS_ACCESS_KEY_ID"
-	EnvSecretAccessKey       = "AWS_SECRET_ACCESS_KEY"
-	EnvSessionToken          = "AWS_SESSION_TOKEN"
-	EnvMountpointCacheKey    = "UNSTABLE_MOUNTPOINT_CACHE_KEY"
+	EnvRegion                          = "AWS_REGION"
+	EnvDefaultRegion                   = "AWS_DEFAULT_REGION"
+	EnvSTSRegionalEndpoints            = "AWS_STS_REGIONAL_ENDPOINTS"
+	EnvMaxAttempts                     = "AWS_MAX_ATTEMPTS"
+	EnvProfile                         = "AWS_PROFILE"
+	EnvConfigFile                      = "AWS_CONFIG_FILE"
+	EnvSharedCredentialsFile           = "AWS_SHARED_CREDENTIALS_FILE"
+	EnvRoleARN                         = "AWS_ROLE_ARN"
+	EnvWebIdentityTokenFile            = "AWS_WEB_IDENTITY_TOKEN_FILE"
+	EnvContainerAuthorizationTokenFile = "AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE"
+	EnvContainerCredentialsFullURI     = "AWS_CONTAINER_CREDENTIALS_FULL_URI"
+	EnvEC2MetadataDisabled             = "AWS_EC2_METADATA_DISABLED"
+	EnvAccessKeyID                     = "AWS_ACCESS_KEY_ID"
+	EnvSecretAccessKey                 = "AWS_SECRET_ACCESS_KEY"
+	EnvSessionToken                    = "AWS_SESSION_TOKEN"
+	EnvMountpointCacheKey              = "UNSTABLE_MOUNTPOINT_CACHE_KEY"
 )
 
 // Key represents an environment variable name.

tests/e2e-kubernetes/e2e_test.go

@@ -23,6 +23,7 @@ func init() {
 
 	flag.StringVar(&CommitId, "commit-id", "local", "commit id will be used to name buckets")
 	flag.StringVar(&BucketRegion, "bucket-region", "us-east-1", "region where temporary buckets will be created")
+	flag.StringVar(&ClusterName, "cluster-name", "", "name of the cluster")
 	flag.StringVar(&BucketPrefix, "bucket-prefix", "local", "prefix for temporary buckets")
 	flag.BoolVar(&Performance, "performance", false, "run performance tests")
 	flag.BoolVar(&IMDSAvailable, "imds-available", false, "indicates whether instance metadata service is available")
@@ -31,6 +32,7 @@ func init() {
 
 	s3client.DefaultRegion = BucketRegion
 	custom_testsuites.DefaultRegion = BucketRegion
+	custom_testsuites.ClusterName = ClusterName
 	custom_testsuites.IMDSAvailable = IMDSAvailable
 	custom_testsuites.IsPodMounter = IsPodMounter
 }

tests/e2e-kubernetes/go.mod

@@ -3,7 +3,7 @@ module github.com/awslabs/aws-s3-csi-driver/tests/e2e-kubernetes
 go 1.24
 
 require (
-	github.com/aws/aws-sdk-go-v2 v1.30.5
+	github.com/aws/aws-sdk-go-v2 v1.36.3
 	github.com/aws/aws-sdk-go-v2/config v1.27.33
 	github.com/aws/aws-sdk-go-v2/service/iam v1.34.3
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.47.3
@@ -29,17 +29,19 @@ require (
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.3 // indirect
 	github.com/aws/aws-sdk-go-v2/credentials v1.17.32 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.13 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.17 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.17 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.8 // indirect
+	github.com/aws/aws-sdk-go-v2/service/eks v1.64.0
+	github.com/aws/aws-sdk-go-v2/service/eksauth v1.8.2
 	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.4 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.8 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.19 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.8 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sso v1.22.7 // indirect
 	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.7 // indirect
-	github.com/aws/smithy-go v1.20.4 // indirect
+	github.com/aws/smithy-go v1.22.2 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/blang/semver/v4 v4.0.0 // indirect
 	github.com/cenkalti/backoff/v4 v4.3.0 // indirect