Add proxy support (HTTPS_PROXY, NO_PROXY) via volume attributes (#663)

*Issue #, if available:*
Relates #533, #535

*Description of changes:*
Support injecting NO_PROXY, HTTPS_PROXY to mount-s3 process


By submitting this pull request, I confirm that you can use, modify,
copy, and redistribute this contribution, under the terms of your
choice.

---------

Signed-off-by: phuhung273 <phuhung.tranpham@gmail.com>
Co-authored-by: Yerzhan Mazhkenov <20302932+yerzhan7@users.noreply.github.com>

Author: phuhung273

docs/CONFIGURATION.md

@@ -62,6 +62,12 @@ spec:
       mountpointContainerResourcesLimitsCpu: 500m
       mountpointContainerResourcesLimitsMemory: 1Gi
 
+      # ----- ENVIRONMENT VARIABLE CONFIGURATION -----
+      # Optional: Pass extra environment variables to the mount-s3 process. Default: none.
+      # Supported variables: HTTPS_PROXY, NO_PROXY
+      mountpointEnv.HTTPS_PROXY: "http://proxy.example.com:3128"
+      mountpointEnv.NO_PROXY: "sts.us-east-1.amazonaws.com"
+
 ---
 apiVersion: v1
 kind: PersistentVolumeClaim

examples/kubernetes/static_provisioning/README.md

@@ -10,6 +10,7 @@ This example shows how to make a static provisioned Mountpoint for S3 persistent
 - `caching.yaml` - shows how to configure mountpoint to use a cache directory. See the [Mountpoint documentation](https://github.com/awslabs/mountpoint-s3/blob/main/doc/CONFIGURATION.md#caching-configuration) for more details on caching options. Please thumbs up [#11](https://github.com/awslabs/mountpoint-s3-csi-driver/issues/141) or add details about your use case if you want improvements in this area.
 - `kms_sse.yaml` - demonstrates using SSE-KMS encryption with a customer supplied key id. See the [Mountpoint documentation](https://github.com/awslabs/mountpoint-s3/blob/main/doc/CONFIGURATION.md#data-encryption) for more details.
 - `aws_max_attempts.yaml` - configure the number of retries for requests to S3. This option is passed to Mountpoint as the `AWS_MAX_ATTEMPTS` environment variable. See the [Mountpoint configuration documentation](https://github.com/awslabs/mountpoint-s3/blob/main/doc/CONFIGURATION.md#other-s3-bucket-configuration) for more details.
+- `proxy.yaml` - configure HTTPS_PROXY, NO_PROXY environment variables for mountpoint behind proxy. See the [Mountpoint documentation](https://github.com/awslabs/mountpoint-s3-csi-driver/blob/main/docs/CONFIGURATION.md#static-provisioning) for more details.
 ## Configure
 ### Edit [Persistent Volume](https://github.com/awslabs/mountpoint-s3-csi-driver/blob/main/examples/kubernetes/static_provisioning/static_provisioning.yaml)
 > Note: This example assumes your S3 bucket has already been created. If you need to create a bucket, follow the [S3 documentation](https://docs.aws.amazon.com/AmazonS3/latest/userguide/creating-bucket.html) for a general purpose bucket or the [S3 Express One Zone documentation](https://docs.aws.amazon.com/AmazonS3/latest/userguide/directory-bucket-create.html) for a directory bucket.

examples/kubernetes/static_provisioning/proxy.yaml

@@ -0,0 +1,78 @@
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: s3-pv
+spec:
+  capacity:
+    storage: 1Gi
+  accessModes:
+    - ReadWriteMany # Supported options: ReadWriteMany / ReadOnlyMany
+  storageClassName: "" # Required for static provisioning
+  claimRef: # To ensure no other PVCs can claim this PV
+    namespace: default # Namespace is required even though it's in "default" namespace.
+    name: s3-pvc # Name of your PVC
+  mountOptions:
+    - allow-delete
+    - region us-east-1
+  csi:
+    driver: s3.csi.aws.com # Required
+    volumeHandle: s3-csi-driver-volume # Must be unique
+    volumeAttributes:
+      bucketName: s3-csi-driver
+      # Supported variables: HTTPS_PROXY, NO_PROXY
+      mountpointEnv.HTTPS_PROXY: "http://proxy.default.svc.cluster.local:3128"
+      mountpointEnv.NO_PROXY: "sts.us-east-1.amazonaws.com"
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: s3-pvc
+spec:
+  accessModes:
+    - ReadWriteMany # Supported options: ReadWriteMany / ReadOnlyMany
+  storageClassName: "" # Required for static provisioning
+  resources:
+    requests:
+      storage: 1Gi
+  volumeName: s3-pv # Name of your PV
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: proxy
+  labels:
+    app: proxy
+spec:
+  containers:
+    - name: proxy
+      image: ubuntu/squid:latest
+      ports:
+        - containerPort: 3128
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: proxy
+spec:
+  selector:
+    app: proxy
+  ports:
+    - port: 3128
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: s3-app
+spec:
+  containers:
+    - name: app
+      image: ubuntu
+      command: ["/bin/sh"]
+      args: ["-c", "echo 'Hello from the container!' >> /data/$(date -u).txt; tail -f /dev/null"]
+      volumeMounts:
+        - name: persistent-storage
+          mountPath: /data
+  volumes:
+    - name: persistent-storage
+      persistentVolumeClaim:
+        claimName: s3-pvc

pkg/driver/node/envprovider/provider.go

@@ -6,6 +6,7 @@ import (
 	"maps"
 	"os"
 	"slices"
+	"strings"
 )
 
 const (
@@ -25,8 +26,12 @@ const (
 	EnvSecretAccessKey                 = "AWS_SECRET_ACCESS_KEY"
 	EnvSessionToken                    = "AWS_SESSION_TOKEN"
 	EnvMountpointCacheKey              = "UNSTABLE_MOUNTPOINT_CACHE_KEY"
+	EnvHTTPSProxy                      = "HTTPS_PROXY"
+	EnvNoProxy                         = "NO_PROXY"
 )
 
+const mountpointEnvPrefix = "mountpointEnv."
+
 // Key represents an environment variable name.
 type Key = string
 
@@ -44,6 +49,12 @@ var envAllowlist = []Key{
 	EnvSTSRegionalEndpoints,
 }
 
+// userEnvAllowlist is the list of environment variables allowed to be configured by user.
+var userEnvAllowlist = []Key{
+	EnvHTTPSProxy,
+	EnvNoProxy,
+}
+
 // Region returns detected region from environment variables `AWS_REGION` or `AWS_DEFAULT_REGION`.
 // It returns an empty string if both is unset.
 func Region() Value {
@@ -66,6 +77,21 @@ func Default() Environment {
 	return environment
 }
 
+func ParseUserEnvFromVolumeContext(volumeCtx map[string]string) (Environment, error) {
+	env := Environment{}
+	for key, value := range volumeCtx {
+		envName, ok := strings.CutPrefix(key, mountpointEnvPrefix)
+		if !ok {
+			continue
+		}
+		if !slices.Contains(userEnvAllowlist, envName) {
+			return nil, fmt.Errorf("environment variable not allowed: %s (allowed: %v)", key, userEnvAllowlist)
+		}
+		env.Set(envName, value)
+	}
+	return env, nil
+}
+
 // List returns a sorted slice of environment variables in "KEY=VALUE" format.
 func (env Environment) List() []string {
 	list := []string{}

pkg/driver/node/envprovider/provider_test.go

@@ -1,6 +1,7 @@
 package envprovider_test
 
 import (
+	"strings"
 	"testing"
 
 	"github.com/awslabs/mountpoint-s3-csi-driver/pkg/driver/node/envprovider"
@@ -269,3 +270,55 @@ func TestMergingEnvironments(t *testing.T) {
 		})
 	}
 }
+
+func TestParseUserEnvFromVolumeContext(t *testing.T) {
+	testCases := []struct {
+		name      string
+		volumeCtx map[string]string
+		want      envprovider.Environment
+		errMsg    string
+	}{
+		{
+			name: "correct",
+			volumeCtx: map[string]string{
+				"mountpointEnv.HTTPS_PROXY": "proxy:3128",
+				"mountpointEnv.NO_PROXY":    "169.254.169.254",
+			},
+			want: envprovider.Environment{
+				"HTTPS_PROXY": "proxy:3128",
+				"NO_PROXY":    "169.254.169.254",
+			},
+		},
+		{
+			name: "correct, skipping invalid prefix",
+			volumeCtx: map[string]string{
+				"mountpointEnv.HTTPS_PROXY":    "proxy:3128",
+				"mountpointEnvvvv.HTTPS_PROXY": "invalidPrefix",
+			},
+			want: envprovider.Environment{
+				"HTTPS_PROXY": "proxy:3128",
+			},
+		},
+		{
+			name: "failed, environment not allowed",
+			volumeCtx: map[string]string{
+				"mountpointEnv.FOO": "BAR",
+			},
+			errMsg: "environment variable not allowed: mountpointEnv.FOO",
+		},
+	}
+
+	for _, testCase := range testCases {
+		t.Run(testCase.name, func(t *testing.T) {
+			userEnv, err := envprovider.ParseUserEnvFromVolumeContext(testCase.volumeCtx)
+			if testCase.errMsg != "" {
+				if !strings.Contains(err.Error(), testCase.errMsg) {
+					t.Errorf("Expected error message %q, but got %q", testCase.errMsg, err.Error())
+				}
+			} else {
+				assert.NoError(t, err)
+				assert.Equals(t, testCase.want, userEnv)
+			}
+		})
+	}
+}

pkg/driver/node/mounter/fake_mounter.go

@@ -4,13 +4,14 @@ import (
 	"context"
 
 	"github.com/awslabs/mountpoint-s3-csi-driver/pkg/driver/node/credentialprovider"
+	"github.com/awslabs/mountpoint-s3-csi-driver/pkg/driver/node/envprovider"
 	"github.com/awslabs/mountpoint-s3-csi-driver/pkg/mountpoint"
 )
 
 type FakeMounter struct{}
 
 func (m *FakeMounter) Mount(ctx context.Context, bucketName string, target string,
-	credentialCtx credentialprovider.ProvideContext, args mountpoint.Args, fsGroup string) error {
+	credentialCtx credentialprovider.ProvideContext, args mountpoint.Args, fsGroup string, userEnv envprovider.Environment) error {
 	return nil
 }
 

pkg/driver/node/mounter/mocks/mock_mount.go

@@ -6,12 +6,13 @@ import (
 	"path/filepath"
 
 	"github.com/awslabs/mountpoint-s3-csi-driver/pkg/driver/node/credentialprovider"
+	"github.com/awslabs/mountpoint-s3-csi-driver/pkg/driver/node/envprovider"
 	"github.com/awslabs/mountpoint-s3-csi-driver/pkg/mountpoint"
 )
 
 // Mounter is an interface for mount operations
 type Mounter interface {
-	Mount(ctx context.Context, bucketName string, target string, credentialCtx credentialprovider.ProvideContext, args mountpoint.Args, fsGroup string) error
+	Mount(ctx context.Context, bucketName string, target string, credentialCtx credentialprovider.ProvideContext, args mountpoint.Args, fsGroup string, userEnv envprovider.Environment) error
 	Unmount(ctx context.Context, target string, credentialCtx credentialprovider.CleanupContext) error
 	IsMountPoint(target string) (bool, error)
 }

pkg/driver/node/mounter/mounter.go

@@ -102,7 +102,7 @@ func NewPodMounter(
 //
 // If Mountpoint is already mounted at `target`, it will return early at step 3 to ensure credentials are up-to-date.
 // If Mountpoint is already mounted at `source`, it will skip steps 4-7 and only perform bind mount to `target`.
-func (pm *PodMounter) Mount(ctx context.Context, bucketName string, target string, credentialCtx credentialprovider.ProvideContext, args mountpoint.Args, fsGroup string) error {
+func (pm *PodMounter) Mount(ctx context.Context, bucketName string, target string, credentialCtx credentialprovider.ProvideContext, args mountpoint.Args, fsGroup string, userEnv envprovider.Environment) error {
 	volumeName, err := pm.volumeNameFromTargetPath(target)
 	if err != nil {
 		return fmt.Errorf("Failed to extract volume name from %q: %w", target, err)
@@ -163,7 +163,7 @@ func (pm *PodMounter) Mount(ctx context.Context, bucketName string, target strin
 	}
 
 	if !isSourceMountPoint {
-		err = pm.mountS3AtSource(ctx, source, pod, podPath, bucketName, credEnv, authenticationSource, args)
+		err = pm.mountS3AtSource(ctx, source, pod, podPath, bucketName, credEnv, userEnv, authenticationSource, args)
 		if err != nil {
 			return fmt.Errorf("Failed to mount at source %q: %w. %s", source, err, pm.helpMessageForGettingMountpointLogs(pod))
 		}
@@ -194,6 +194,7 @@ func (pm *PodMounter) Mount(ctx context.Context, bucketName string, target strin
 //   - podPath: Base path for Pod-specific files
 //   - bucketName: Name of the S3 bucket to mount
 //   - credEnv: Environment variables related to AWS credentials
+//   - userEnv: Environment variables provided by user
 //   - authenticationSource: Authentication source from PV volume attribute
 //   - args: Mountpoint arguments
 //
@@ -208,9 +209,13 @@ func (pm *PodMounter) Mount(ctx context.Context, bucketName string, target strin
 //
 // If any step fails, it ensures cleanup by unmounting the source path.
 func (pm *PodMounter) mountS3AtSource(ctx context.Context, source string, mpPod *corev1.Pod, podPath string,
-	bucketName string, credEnv envprovider.Environment, authenticationSource credentialprovider.AuthenticationSource,
+	bucketName string, credEnv envprovider.Environment, userEnv envprovider.Environment, authenticationSource credentialprovider.AuthenticationSource,
 	args mountpoint.Args) error {
-	env := envprovider.Default()
+
+	// Build environment with precedence (highest wins): credEnv > Default() > userEnv
+	env := envprovider.Environment{}
+	env.Merge(userEnv)
+	env.Merge(envprovider.Default())
 	env.Merge(credEnv)
 
 	// Move `--aws-max-attempts` to env if provided