Use hard-coded kubelet path for s3-plugin container (#656)

*Issue #577 *

**Description**

On MicroK8s (snap install), kubelet uses
`/var/snap/microk8s/common/var/lib/kubelet` as its root directory. In
CSI Driver v2, Unix socket paths built from this base exceed Linux's
108-byte `UNIX_PATH_MAX` limit, causing bind() / connect() to fail with
invalid argument. The driver either fails to register with kubelet or
fails to communicate with Mountpoint pods, making S3 mounts impossible.
This affects any Kubernetes distribution with a long kubelet path.

**Fix**
Hard-code the container-side mountPath to `/var/lib/kubelet `regardless
of the host path. The host path remains
configurable via `node.kubeletPath` (used as the volume source), but
inside the container all paths are short.

Replace the single `KUBELET_PATH` env var with:
- `CONTAINER_KUBELET_PATH`=`/var/lib/kubelet `— path as seen inside the
container
- `HOST_KUBELET_PATH` — path as seen on the host (from node.kubeletPath)

Add `KubeletHostPathToContainerPath()` to translate host paths (received
from kubelet RPCs) to container paths in both `NodePublishVolume` and
`NodeUnpublishVolume`.

**Verification**
Tested on Ubuntu 22.04 with MicroK8s (snap, channel 1.32) and
node.kubeletPath=/var/snap/microk8s/common/var/lib/kubelet.

Without the fix, the driver fails to register as reported in Issue #577
— the registration socket path exceeds Linux's 108-byte limit.

With the fix — driver registers successfully and all 3 node containers
are running:
```
$ microk8s kubectl get pods -n kube-system | grep s3-csi
s3-csi-controller-558c76774b-4mc8f         1/1     Running   0          5m
s3-csi-node-spmkn                          3/3     Running   0          5m
```

```
$ microk8s kubectl logs -n kube-system -l app=s3-csi-node -c node-driver-registrar
I0322 14:02:26.423604       1 main.go:145] "Version" version="v2.16.0-eksbuild.1"
I0322 14:02:26.428943       1 node_register.go:56] "Starting Registration Server" socketPath="/registration/s3.csi.aws.com-reg.sock"
I0322 14:02:26.429167       1 node_register.go:66] "Registration Server started" socketPath="/registration/s3.csi.aws.com-reg.sock"
I0322 14:02:26.965157       1 main.go:102] "Received NotifyRegistrationStatus call" status="plugin_registered:true"
```

S3 mount works — workload pod writes to the bucket:
```
$ microk8s kubectl exec s3-app -- ls -la /data/
$ aws s3 ls tadiwaom-s3-test/
...
2026-03-22 21:30:34         26 Sun Mar 22 21:30:32 UTC 2026.txt
```

**Testing**
Manual Testing with E2E tests passing: (rosa was not set up in my CI) 

https://github.com/tadiwa-aizen/mountpoint-s3-csi-driver/actions/runs/23405969976

By submitting this pull request, I confirm that you can use, modify,
copy, and redistribute this contribution, under the terms of your
choice.

---------

Signed-off-by: Tadiwa Magwenzi <tadiwaom@amazon.com>
Co-authored-by: Yerzhan Mazhkenov <20302932+yerzhan7@users.noreply.github.com>

Author: tadiwa-aizen

charts/aws-mountpoint-s3-csi-driver/templates/node.yaml

@@ -82,15 +82,17 @@ spec:
             - --v={{ .Values.node.logLevel }}
           env:
             - name: CSI_ENDPOINT
-              value: unix:{{ trimSuffix "/" .Values.node.kubeletPath }}/plugins/s3.csi.aws.com/csi.sock
-            - name: KUBELET_PATH
-              value: {{ .Values.node.kubeletPath }}
+              value: unix:/var/lib/kubelet/plugins/s3.csi.aws.com/csi.sock
             - name: CSI_NODE_NAME
               valueFrom:
                 fieldRef:
                   fieldPath: spec.nodeName
             - name: HOST_PLUGIN_DIR
-              value: {{ trimSuffix "/" .Values.node.kubeletPath }}/plugins/s3.csi.aws.com/
+              value: /var/lib/kubelet/plugins/s3.csi.aws.com/
+            - name: CONTAINER_KUBELET_PATH
+              value: /var/lib/kubelet
+            - name: HOST_KUBELET_PATH
+              value: {{ .Values.node.kubeletPath }}
             - name: SUPPORT_LEGACY_SYSTEMD_MOUNTS
               value: "{{ .Values.supportLegacySystemDMounts }}"
             - name: MOUNTPOINT_NAMESPACE
@@ -121,7 +123,7 @@ spec:
             {{- end }}
           volumeMounts:
             - name: kubelet-dir
-              mountPath: {{ .Values.node.kubeletPath }}
+              mountPath: /var/lib/kubelet
               mountPropagation: Bidirectional
           ports:
             - name: healthz

deploy/kubernetes/base/node.yaml

@@ -62,14 +62,16 @@ spec:
           env:
             - name: CSI_ENDPOINT
               value: unix:/var/lib/kubelet/plugins/s3.csi.aws.com/csi.sock
-            - name: KUBELET_PATH
-              value: /var/lib/kubelet
             - name: CSI_NODE_NAME
               valueFrom:
                 fieldRef:
                   fieldPath: spec.nodeName
             - name: HOST_PLUGIN_DIR
               value: /var/lib/kubelet/plugins/s3.csi.aws.com/
+            - name: CONTAINER_KUBELET_PATH
+              value: /var/lib/kubelet
+            - name: HOST_KUBELET_PATH
+              value: /var/lib/kubelet
             - name: SUPPORT_LEGACY_SYSTEMD_MOUNTS
               value: "true"
             - name: MOUNTPOINT_NAMESPACE

pkg/driver/node/mounter/pod_mounter.go

@@ -79,7 +79,7 @@ func NewPodMounter(
 		s3paCache:         s3paCache,
 		credProvider:      credProvider,
 		mount:             mount,
-		kubeletPath:       util.KubeletPath(),
+		kubeletPath:       util.ContainerKubeletPath(),
 		mountSyscall:      mountSyscall,
 		bindMountSyscall:  bindMountSyscall,
 		kubernetesVersion: kubernetesVersion,

pkg/driver/node/mounter/pod_mounter_test.go

@@ -78,7 +78,7 @@ func setup(t *testing.T) *testCtx {
 	parentDir, err := filepath.EvalSymlinks(filepath.Dir(kubeletPath))
 	assert.NoError(t, err)
 	kubeletPath = filepath.Join(parentDir, filepath.Base(kubeletPath))
-	t.Setenv("KUBELET_PATH", kubeletPath)
+	t.Setenv("CONTAINER_KUBELET_PATH", kubeletPath)
 
 	// Chdir to `kubeletPath` so `mountoptions.{Recv, Send}` can use relative paths to Unix sockets
 	// to overcome `bind: invalid argument`.

pkg/driver/node/mounter/pod_unmounter.go

@@ -49,7 +49,7 @@ func NewPodUnmounter(
 	return &PodUnmounter{
 		nodeID:       nodeID,
 		mount:        mount,
-		kubeletPath:  util.KubeletPath(),
+		kubeletPath:  util.ContainerKubeletPath(),
 		podWatcher:   podWatcher,
 		credProvider: credProvider,
 	}

pkg/driver/node/mounter/pod_unmounter_test.go

@@ -193,7 +193,7 @@ func TestHandleS3PodAttachmentUpdate(t *testing.T) {
 			parentDir, err := filepath.EvalSymlinks(filepath.Dir(kubeletPath))
 			assert.NoError(t, err)
 			kubeletPath = filepath.Join(parentDir, filepath.Base(kubeletPath))
-			t.Setenv("KUBELET_PATH", kubeletPath)
+			t.Setenv("CONTAINER_KUBELET_PATH", kubeletPath)
 			t.Chdir(kubeletPath)
 
 			sourceMountDir := mounter.SourceMountDir(kubeletPath)
@@ -337,7 +337,7 @@ func TestCleanupDanglingMounts(t *testing.T) {
 			parentDir, err := filepath.EvalSymlinks(filepath.Dir(kubeletPath))
 			assert.NoError(t, err)
 			kubeletPath = filepath.Join(parentDir, filepath.Base(kubeletPath))
-			t.Setenv("KUBELET_PATH", kubeletPath)
+			t.Setenv("CONTAINER_KUBELET_PATH", kubeletPath)
 			t.Chdir(kubeletPath)
 
 			sourceMountDir := mounter.SourceMountDir(kubeletPath)

pkg/driver/node/node.go

@@ -21,7 +21,6 @@ import (
 	"maps"
 	"os"
 	"strconv"
-	"strings"
 
 	"github.com/container-storage-interface/spec/lib/go/csi"
 	"google.golang.org/grpc/codes"
@@ -39,8 +38,6 @@ import (
 	"github.com/awslabs/mountpoint-s3-csi-driver/pkg/util"
 )
 
-var kubeletPath = util.KubeletPath()
-
 var (
 	nodeCaps = []csi.NodeServiceCapability_RPC_Type{
 		csi.NodeServiceCapability_RPC_VOLUME_MOUNT_GROUP,
@@ -96,13 +93,17 @@ func (ns *S3NodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePubl
 		return nil, status.Error(codes.InvalidArgument, "Bucket name not provided")
 	}
 
-	target := req.GetTargetPath()
-	if len(target) == 0 {
+	targetHost := req.GetTargetPath()
+	if len(targetHost) == 0 {
 		return nil, status.Error(codes.InvalidArgument, "Target path not provided")
 	}
 
-	if !strings.HasPrefix(target, kubeletPath) {
-		klog.Errorf("NodePublishVolume: target path %q is not in kubelet path %q. This might cause mounting issues, please ensure you have correct kubelet path configured.", target, kubeletPath)
+	// Translate target path from host format to container format if needed.
+	// Example error: Failed to translate target path "/opt/kubernetes/kubelet/pods/abcd-1234/volumes/kubernetes.io~csi/s3-pv/mount":
+	//   path "/opt/kubernetes/kubelet/pods/abcd-1234/volumes/kubernetes.io~csi/s3-pv/mount" does not start with host kubelet path "/var/lib/kubelet"
+	targetContainer, err := util.KubeletHostPathToContainerPath(targetHost)
+	if err != nil {
+		return nil, status.Errorf(codes.InvalidArgument, "Failed to translate target path %q: %v", targetHost, err)
 	}
 
 	volCap := req.GetVolumeCapability()
@@ -190,15 +191,15 @@ func (ns *S3NodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePubl
 		}
 	}
 
-	klog.V(4).Infof("NodePublishVolume: mounting %s at %s with options %v", bucket, target, args.SortedList())
+	klog.V(4).Infof("NodePublishVolume: mounting %s at %s with options %v", bucket, targetContainer, args.SortedList())
 
 	credentialCtx := credentialProvideContextFromPublishRequest(req, args)
 
-	if err := ns.Mounter.Mount(ctx, bucket, target, credentialCtx, args, fsGroup); err != nil {
-		os.Remove(target)
-		return nil, status.Errorf(codes.Internal, "Could not mount %q at %q: %v", bucket, target, err)
+	if err := ns.Mounter.Mount(ctx, bucket, targetContainer, credentialCtx, args, fsGroup); err != nil {
+		os.Remove(targetContainer)
+		return nil, status.Errorf(codes.Internal, "Could not mount %q at %q: %v", bucket, targetContainer, err)
 	}
-	klog.V(4).Infof("NodePublishVolume: %s was mounted", target)
+	klog.V(4).Infof("NodePublishVolume: %s was mounted", targetContainer)
 
 	return &csi.NodePublishVolumeResponse{}, nil
 }
@@ -211,32 +212,37 @@ func (ns *S3NodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUn
 		return nil, status.Error(codes.InvalidArgument, "Volume ID not provided")
 	}
 
-	target := req.GetTargetPath()
-	if len(target) == 0 {
+	targetHost := req.GetTargetPath()
+	if len(targetHost) == 0 {
 		return nil, status.Error(codes.InvalidArgument, "Target path not provided")
 	}
 
-	mounted, err := ns.Mounter.IsMountPoint(target)
+	targetContainer, err := util.KubeletHostPathToContainerPath(targetHost)
+	if err != nil {
+		return nil, status.Errorf(codes.InvalidArgument, "Failed to translate target path %q: %v", targetHost, err)
+	}
+
+	mounted, err := ns.Mounter.IsMountPoint(targetContainer)
 	if err != nil && os.IsNotExist(err) {
-		klog.V(4).Infof("NodeUnpublishVolume: target path %s does not exist, skipping unmount", target)
+		klog.V(4).Infof("NodeUnpublishVolume: target path %s does not exist, skipping unmount", targetContainer)
 		return &csi.NodeUnpublishVolumeResponse{}, nil
 	} else if err != nil && mount.IsCorruptedMnt(err) {
-		klog.V(4).Infof("NodeUnpublishVolume: target path %s is corrupted: %v, will try to unmount", target, err)
+		klog.V(4).Infof("NodeUnpublishVolume: target path %s is corrupted: %v, will try to unmount", targetContainer, err)
 		mounted = true
 	} else if err != nil {
-		return nil, status.Errorf(codes.Internal, "Could not unmount %q: %v", target, err)
+		return nil, status.Errorf(codes.Internal, "Could not unmount %q: %v", targetContainer, err)
 	}
 	if !mounted {
-		klog.V(4).Infof("NodeUnpublishVolume: target path %s not mounted, skipping unmount", target)
+		klog.V(4).Infof("NodeUnpublishVolume: target path %s not mounted, skipping unmount", targetContainer)
 		return &csi.NodeUnpublishVolumeResponse{}, nil
 	}
 
 	credentialCtx := credentialCleanupContextFromUnpublishRequest(req)
 
-	klog.V(4).Infof("NodeUnpublishVolume: unmounting %s", target)
-	err = ns.Mounter.Unmount(ctx, target, credentialCtx)
+	klog.V(4).Infof("NodeUnpublishVolume: unmounting %s", targetContainer)
+	err = ns.Mounter.Unmount(ctx, targetContainer, credentialCtx)
 	if err != nil {
-		return nil, status.Errorf(codes.Internal, "Could not unmount %q: %v", target, err)
+		return nil, status.Errorf(codes.Internal, "Could not unmount %q: %v", targetContainer, err)
 	}
 
 	return &csi.NodeUnpublishVolumeResponse{}, nil

pkg/driver/node/node_test.go

@@ -48,7 +48,7 @@ func TestNodePublishVolume(t *testing.T) {
 				Mode: csi.VolumeCapability_AccessMode_MULTI_NODE_MULTI_WRITER,
 			},
 		}
-		targetPath = "/target/path"
+		targetPath = "/var/lib/kubelet/target/path"
 	)
 	testCases := []struct {
 		name     string
@@ -379,7 +379,7 @@ func TestNodePublishVolumeForPodMounter(t *testing.T) {
 	var (
 		volumeId   = "test-volume-id"
 		bucketName = "test-bucket-name"
-		targetPath = "/target/path"
+		targetPath = "/var/lib/kubelet/target/path"
 	)
 	testCases := []struct {
 		name     string
@@ -602,7 +602,7 @@ func TestNodePublishVolumeMaxCacheSizeInjection(t *testing.T) {
 	var (
 		volumeId   = "test-volume-id"
 		bucketName = "test-bucket-name"
-		targetPath = "/target/path"
+		targetPath = "/var/lib/kubelet/target/path"
 	)
 
 	testCases := []struct {
@@ -792,7 +792,7 @@ func TestNodePublishVolumeMaxCacheSizeInjection(t *testing.T) {
 func TestNodeUnpublishVolume(t *testing.T) {
 	var (
 		volumeId   = "test-volume-id"
-		targetPath = "/target/path"
+		targetPath = "/var/lib/kubelet/target/path"
 	)
 	testCases := []struct {
 		name     string

pkg/util/kubelet.go

@@ -1,15 +1,53 @@
 package util
 
-import "os"
+import (
+	"fmt"
+	"os"
+	"strings"
+)
 
 const defaultKubeletPath = "/var/lib/kubelet"
 
-// KubeletPath returns path of the kubelet.
-// It looks for `KUBELET_PATH` variable, and returns a default path if its not defined.
-func KubeletPath() string {
-	kubeletPath := os.Getenv("KUBELET_PATH")
+// ContainerKubeletPath returns the kubelet path as seen from inside the container.
+// It looks for `CONTAINER_KUBELET_PATH` variable, and returns a default path if its not defined.
+func ContainerKubeletPath() string {
+	kubeletPath := os.Getenv("CONTAINER_KUBELET_PATH")
 	if kubeletPath == "" {
 		return defaultKubeletPath
 	}
 	return kubeletPath
 }
+
+// HostKubeletPath returns the kubelet path as seen on the host.
+// It looks for `HOST_KUBELET_PATH` variable, and returns a default path if its not defined.
+func HostKubeletPath() string {
+	kubeletPath := os.Getenv("HOST_KUBELET_PATH")
+	if kubeletPath == "" {
+		return defaultKubeletPath
+	}
+	return kubeletPath
+}
+
+// KubeletHostPathToContainerPath translates a path from host kubelet path to container kubelet path.
+// This is useful when the kubelet path on the host is different from the kubelet path in the container.
+// For example, in MicroK8s, the host path is /var/snap/microk8s/common/var/lib/kubelet
+// but the container sees it as /var/lib/kubelet.
+func KubeletHostPathToContainerPath(path string) (string, error) {
+	hostPath := HostKubeletPath()
+	containerPath := ContainerKubeletPath()
+
+	// If paths are the same, no translation needed
+	if hostPath == containerPath {
+		if strings.HasPrefix(path, containerPath) {
+			return path, nil
+		}
+		return "", fmt.Errorf("path %q does not start with kubelet path %q", path, containerPath)
+	}
+
+	// If the path starts with the host kubelet path, replace it with the container path
+	if strings.HasPrefix(path, hostPath) {
+		return strings.Replace(path, hostPath, containerPath, 1), nil
+	}
+
+	return "", fmt.Errorf("path %q does not start with host kubelet path %q", path, hostPath)
+}