Merge branch 'main' into add-retry-for-new-eks-pod-identity-association-test

Author: yerzhan7

.github/matrix.yaml

@@ -13,22 +13,13 @@ matrix:
       "Ubuntu2404",
     ]
   kubernetes-version:
-    ["1.29.8", "1.30.4", "1.31.0", "1.32.1", "1.33.2", "1.34.1", "1.35.0"]
+    ["1.30.4", "1.31.0", "1.32.1", "1.33.2", "1.34.1", "1.35.0"]
   include:
-    # Ubuntu2004 is only supported on EKS <= 1.29.
-    # See https://eksctl.io/usage/custom-ami-support/?h=ubuntu#setting-the-node-ami-family.
-    - cluster-type: "eksctl"
-      arch: "arm"
-      family: "Ubuntu2004"
-      kubernetes-version: "1.29.8"
     # Enable enforcing mode for SELinux in AL2023, it's easier to list it in "include"
     # field rather than trying to exclude all other variants.
     - family: "AmazonLinux2023"
       selinux-mode: "enforcing"
   exclude:
-    - cluster-type: "eksctl"
-      arch: "arm"
-      family: "Bottlerocket"
     # AL2 is not supported by Kubernetes 1.33.
     - cluster-type: "eksctl"
       family: "AmazonLinux2"
@@ -54,9 +45,6 @@ matrix:
       kubernetes-version: "1.35.0"
     # Ubuntu2404 is only supported on EKS >= 1.31.
     # See https://eksctl.io/usage/custom-ami-support/?h=ubuntu#setting-the-node-ami-family.
-    - cluster-type: "eksctl"
-      family: "Ubuntu2404"
-      kubernetes-version: "1.29.8"
     - cluster-type: "eksctl"
       family: "Ubuntu2404"
       kubernetes-version: "1.30.4"

.github/workflows/cleanup-old-buckets.yaml

@@ -0,0 +1,48 @@
+name: Cleanup Old Test Buckets
+
+on:
+  schedule:
+    # Run every Wednesday at 6:00 AM UTC (cron: minute hour day-of-month month day-of-week)
+    - cron: '0 6 * * 3'
+  workflow_dispatch:
+    inputs:
+      days_old:
+        description: Number of days old for buckets to be deleted
+        required: false
+        default: 7
+        type: number
+
+jobs:
+  cleanup-old-buckets:
+    runs-on: ubuntu-latest
+    # this is to prevent the job to run at forked projects
+    if: ${{ ! github.repository.fork }}
+    strategy:
+      matrix:
+        environment: [trusted, untrusted, rosa-trusted, rosa-untrusted]
+    environment: ${{ matrix.environment }}
+    permissions:
+      id-token: write # Required for OIDC authentication with AWS
+      contents: read
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+        with:
+          persist-credentials: false
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v6
+        with:
+          role-to-assume: ${{ vars.IAM_ROLE }}
+          aws-region: ${{ vars.AWS_REGION }}
+          role-duration-seconds: 3600 # 1 hour should be sufficient for cleanup
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+
+      - name: Delete Old Test Buckets
+        env:
+          AWS_REGION: ${{ vars.AWS_REGION }}
+        run: |
+          tests/e2e-kubernetes/scripts/bucket-cleanup.py cleanup \
+              --days-old ${{ github.event.inputs.days_old || '7' }}

.github/workflows/controller-tests.yaml

@@ -4,7 +4,6 @@ on:
   push:
     branches: ["main", "feature/*"]
   pull_request:
-    branches: ["main", "feature/*"]
   merge_group:
     types: ["checks_requested"]
 
@@ -41,6 +40,7 @@ jobs:
           make e2e-controller
 
   post_test:
+    name: "Controller Post Test"
     if: always()
     needs:
       - controller_test

.github/workflows/delete-cluster.yaml

@@ -17,6 +17,8 @@ jobs:
   build_matrix:
     name: Build Matrix
     uses: ./.github/workflows/build_matrix.yaml
+    permissions:
+      contents: read
   delete_cluster:
     needs: ["build_matrix"]
     strategy:

.github/workflows/e2e-rosa-tests.yaml

@@ -69,7 +69,7 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v6
         with:
-          ref: ${{ github.ref }}
+          ref: ${{ inputs.ref }}
           persist-credentials: false
       - name: Set up Go
         uses: actions/setup-go@v6
@@ -168,6 +168,7 @@ jobs:
             OPENSHIFT_CREDS,s3-csi-${{ inputs.environment }}-openshift-credentials
           parse-json-secrets: true
       - name: Login to OpenShift
+        id: openshift_login
         uses: redhat-actions/oc-login@v1
         with:
           openshift_server_url: ${{ env.OPENSHIFT_CREDS_CLUSTER_API_URL }}
@@ -190,7 +191,8 @@ jobs:
           ACTION: "e2e_cleanup"
         run: tests/e2e-kubernetes/scripts/run.sh
       - name: Uninstall the driver
-        if: always()
+        # Skip uninstall if OpenShift login did not succeed - no cluster access
+        if: ${{ always() && steps.openshift_login.conclusion == 'success' }}
         env:
           ACTION: "uninstall_driver"
         run: tests/e2e-kubernetes/scripts/run.sh

.github/workflows/e2e-tests.yaml

@@ -36,6 +36,8 @@ jobs:
   build_matrix:
     name: Build Matrix
     uses: ./.github/workflows/build_matrix.yaml
+    permissions:
+      contents: read
   build:
     runs-on: ubuntu-22.04 # FIXME - https://github.com/actions/runner-images/issues/11471
     environment: ${{ inputs.environment }}
@@ -116,6 +118,7 @@ jobs:
         run: |
           tests/e2e-kubernetes/scripts/run.sh
       - name: Create cluster
+        id: create_cluster
         env:
           ACTION: "create_cluster"
         run: |
@@ -142,7 +145,8 @@ jobs:
         run: |
           tests/e2e-kubernetes/scripts/run.sh
       - name: Uninstall the driver
-        if: always()
+        # Skip uninstall if cluster creation did not succeed - no cluster to communicate with
+        if: ${{ always() && steps.create_cluster.conclusion == 'success' }}
         env:
           ACTION: "uninstall_driver"
         run: |

.github/workflows/helm-publish.yaml

@@ -1,20 +1,24 @@
 name: Helm publish
 
 on:
+  # This job is manually dispatched for now, since we do not have image build fully automated yet.
   workflow_dispatch:
     inputs:
       tag:
-        description: "Tag referencing commit to create release from"
+        description: "Release tag (e.g. v2.0.0)"
+        required: true
+      dry-run:
+        description: "Skip publishing Helm chart"
+        type: boolean
         required: true
+        default: true
 
 jobs:
-  # This job is manually dispatched for now, since we do not have image build fully automated yet.
-  helm:
+  verify-helm-chart:
     runs-on: ubuntu-latest
     permissions:
-      contents: write
-    # Ensure only the latest version of the workflow can run, as this is global for the project
-    if: ${{ github.ref == 'refs/heads/main' }}
+      id-token: write
+    environment: "trusted"
     steps:
       - name: Checkout tag
         uses: actions/checkout@v6
@@ -29,10 +33,6 @@ jobs:
           sparse-checkout: |
             scripts/verify-helm-images.sh
           sparse-checkout-cone-mode: false
-      - name: Configure Git
-        run: |
-          git config user.name "$GITHUB_ACTOR"
-          git config user.email "$GITHUB_ACTOR@users.noreply.github.com"
       - name: Install crane
         run: |
           cd /tmp
@@ -41,8 +41,48 @@ jobs:
           sudo mv crane /usr/local/bin/crane
           sudo chmod +x /usr/local/bin/crane
           crane version
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v6
+        with:
+          role-to-assume: ${{ vars.IAM_ROLE }}
+          aws-region: ${{ vars.AWS_REGION }}
+          inline-session-policy: >-
+            {
+              "Version": "2012-10-17",
+              "Statement": [
+                {
+                  "Sid":"AllowECRAuth",
+                  "Effect":"Allow",
+                  "Action":"ecr:GetAuthorizationToken",
+                  "Resource":"*"
+                },
+                {
+                  "Sid":"AllowECRRead",
+                  "Effect": "Allow",
+                  "Action": "ecr:BatchGetImage",
+                  "Resource": "arn:aws:ecr:us-east-1:602401143452:repository/eks/*"
+                }
+              ]
+            }
       - name: Verify all images exist before publishing
         run: ./main-branch/scripts/verify-helm-images.sh
+  publish-helm-chart:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    needs: [verify-helm-chart]
+    # Ensure only the latest version of this job on main can run for publishing, as this is global for the project
+    if: ${{ github.ref == 'refs/heads/main' && !inputs.dry-run }}
+    steps:
+      - name: Checkout tag
+        uses: actions/checkout@v6
+        with:
+          ref: ${{ inputs.tag }}
+          fetch-depth: 0
+      - name: Configure Git
+        run: |
+          git config user.name "$GITHUB_ACTOR"
+          git config user.email "$GITHUB_ACTOR@users.noreply.github.com"
       - name: Run chart-releaser
         uses: helm/chart-releaser-action@v1.7.0
         env:

.github/workflows/unit-tests.yaml

@@ -4,7 +4,6 @@ on:
   push:
     branches: [ "main", "feature/*" ]
   pull_request:
-    branches: [ "main", "feature/*" ]
   merge_group:
     types: [ "checks_requested" ]
 
@@ -45,7 +44,7 @@ jobs:
       run: make cover
 
     - name: Upload report
-      uses: actions/upload-artifact@v6
+      uses: actions/upload-artifact@v7
       id: uploaded-report
       with:
         name: cover

CHANGELOG.md

@@ -1,7 +1,30 @@
 # Unreleased
 
+# v2.5.0
+
+[Documentation](https://github.com/awslabs/mountpoint-s3-csi-driver/blob/v2.5.0/README.md) 
+
 ### Notable changes
 * Add graceful pod eviction to ensure proper termination order. Mountpoint pods now remain active until all workload pods using the volume have terminated, preventing "Transport endpoint is not connected" errors. ([#693](https://github.com/awslabs/mountpoint-s3-csi-driver/pull/693))
+* Forward `CacheEmptyDirSizeLimit` to Mountpoint as `--max-cache-size` when using `emptyDir` cache with the default storage medium. This fixes issues where Mountpoint cache usage would go over `CacheEmptyDirSizeLimit` and cause eviction of the Mountpoint pod. ([#743](https://github.com/awslabs/mountpoint-s3-csi-driver/pull/743))
+* Add `cluster-autoscaler.kubernetes.io/daemonset-pod: "true"` annotation to Mountpoint Pods to prevent Cluster Autoscaler scale-down blocking. This resolves an issue where Cluster Autoscaler treated MP pods as non-replicated singleton pods, preventing node scale-down. ([#675](https://github.com/awslabs/mountpoint-s3-csi-driver/issues/675))
+* Drop support for Kubernetes 1.29. ([#721](https://github.com/awslabs/mountpoint-s3-csi-driver/pull/721))
+* Add `podLabels` configuration to Helm chart values. ([#714](https://github.com/awslabs/mountpoint-s3-csi-driver/pull/714))
+* Add fallback logic for SA tokens via CSI secrets field. ([#728](https://github.com/awslabs/mountpoint-s3-csi-driver/pull/728))
+* Headroom: Prevent premature workload pod ungating when PVC is unbound. ([#708](https://github.com/awslabs/mountpoint-s3-csi-driver/pull/708))
+* Use hard-coded kubelet path for s3-plugin container. ([#656](https://github.com/awslabs/mountpoint-s3-csi-driver/pull/656))
+* Support Mountpoint [version 1.22.2](https://github.com/awslabs/mountpoint-s3/releases/tag/mountpoint-s3-1.22.2) ([#755](https://github.com/awslabs/mountpoint-s3-csi-driver/pull/755))
+  * Update the internal S3 client to use the latest release of the AWS Common Runtime (CRT) libraries. ([#1778](https://github.com/awslabs/mountpoint-s3/pull/1778))
+
+# v2.4.1
+
+[Documentation](https://github.com/awslabs/mountpoint-s3-csi-driver/blob/v2.4.1/README.md) 
+
+* Support Mountpoint [version 1.22.1](https://github.com/awslabs/mountpoint-s3/releases/tag/mountpoint-s3-1.22.1) ([#733](https://github.com/awslabs/mountpoint-s3-csi-driver/pull/733))
+  * Fix a race condition where concurrent operations after closing a truncated file could result in I/O errors on subsequent reads. The issue was introduced in v1.22.0. ([#1781](https://github.com/awslabs/mountpoint-s3/pull/1781))
+  * Fix incorrect validation of default data cache limit which would cause Mountpoint to preserve less than 5% of available space ([#1779](https://github.com/awslabs/mountpoint-s3/pull/1779))
+* Update go to 1.26.0 and run go fix ([#731](https://github.com/awslabs/mountpoint-s3-csi-driver/pull/731))
+* Update csi-node-driver-registrar and livenessprobe sidecar versions for release 2.4 branch ([#739](https://github.com/awslabs/mountpoint-s3-csi-driver/pull/739))
 
 # v2.4.0
 

Dockerfile

@@ -12,7 +12,7 @@
 #See the License for the specific language governing permissions and
 #limitations under the License.
 
-ARG MOUNTPOINT_VERSION=1.22.0
+ARG MOUNTPOINT_VERSION=1.22.2
 
 # Download the mountpoint tarball and produce an installable directory
 # Building on Amazon Linux 2 because it has an old libc version. libfuse from the os
@@ -30,9 +30,10 @@ RUN MP_ARCH=`echo ${TARGETARCH} | sed s/amd64/x86_64/` && \
     wget -q "https://s3.amazonaws.com/mountpoint-s3-release/${MOUNTPOINT_VERSION}/$MP_ARCH/mount-s3-${MOUNTPOINT_VERSION}-$MP_ARCH.tar.gz.asc" && \
     wget -q https://s3.amazonaws.com/mountpoint-s3-release/public_keys/KEYS
 
-# Import the key and validate it has the fingerprint we expect
+# Import the key and validate it has the fingerprints we expect
 RUN gpg --import KEYS && \
-    (gpg --fingerprint mountpoint-s3@amazon.com | grep "673F E406 1506 BB46 9A0E  F857 BE39 7A52 B086 DA5A")
+    (gpg --fingerprint mountpoint-s3@amazon.com | grep "8AEF E705 EBE3 29C0 948C  75A6 6F1C 3B3A EF4B 030B") && \
+    (gpg --fingerprint mountpoint-s3@amazon.com | grep "673F E406 1506 BB46 9A0E  F857 BE39 7A52 B086 DA5A") # older key
 
 # Verify the downloaded tarball, extract it, and fixup the binary
 RUN MP_ARCH=`echo ${TARGETARCH} | sed s/amd64/x86_64/` && \
@@ -43,7 +44,7 @@ RUN MP_ARCH=`echo ${TARGETARCH} | sed s/amd64/x86_64/` && \
     patchelf --set-rpath '$ORIGIN' /mountpoint-s3/bin/mount-s3
 
 # Build driver. Use BUILDPLATFORM not TARGETPLATFORM for cross compilation
-FROM --platform=$BUILDPLATFORM public.ecr.aws/eks-distro-build-tooling/golang:1.25.6 as builder
+FROM --platform=$BUILDPLATFORM public.ecr.aws/eks-distro-build-tooling/golang:1.26.1 as builder
 ARG TARGETARCH
 
 WORKDIR /go/src/github.com/awslabs/mountpoint-s3-csi-driver