E2E: Add waiting for MP Pods to be cleaned up in AfterSuite (#758)

*Issue #, if available:* N/A

*Description of changes:* Only MP Pod sharing tests in the suite
properly wait for MP Pod cleanup, which can take up to 4min in case we
need to rely on stale attachment cleaner in controller (due to eventual
consistency issue in K8s controller runtime framework). When
`uninstall_driver` runs, helm deletes the release and the `mount-s3`
namespace starts terminating. If Mountpoint pods are slow to terminate
(no controller/stale-attachment-cleaner to clean them up since it was
just deleted), the namespace gets stuck in `Terminating`. On the next CI
run, the `helm upgrade --install` fails because Kubernetes refuses to
create resources in the terminating `mount-s3` namespace.

Example:
https://github.com/awslabs/mountpoint-s3-csi-driver/actions/runs/23316182156/job/68086361510?pr=752

```
Error: failed to create resource: roles.rbac.authorization.k8s.io "s3-csi-driver-controller-role" is forbidden: unable to create new content in namespace mount-s3 because it is being terminated
```

The fix: Add waiting for MP Pods to be cleaned up properly in AfterSuite
by stale attachment cleaner in controller.

By submitting this pull request, I confirm that you can use, modify,
copy, and redistribute this contribution, under the terms of your
choice.

Author: yerzhan7

tests/e2e-kubernetes/e2e_test.go

@@ -1,14 +1,17 @@
 package e2e
 
 import (
+	"context"
 	"flag"
 	"testing"
+	"time"
 
 	"github.com/awslabs/mountpoint-s3-csi-driver/tests/e2e-kubernetes/s3client"
 	custom_testsuites "github.com/awslabs/mountpoint-s3-csi-driver/tests/e2e-kubernetes/testsuites"
 
 	ginkgo "github.com/onsi/ginkgo/v2"
 	"github.com/onsi/gomega"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	f "k8s.io/kubernetes/test/e2e/framework"
 	"k8s.io/kubernetes/test/e2e/storage/framework"
 	"k8s.io/kubernetes/test/e2e/storage/testsuites"
@@ -81,6 +84,33 @@ func getCSITestSuites() []func() framework.TestSuite {
 	return suites
 }
 
+// Wait for all Mountpoint pods in mount-s3 namespace to be cleaned up after tests complete.
+// This ensures the mount-s3 namespace is not stuck with stale pods when the driver is uninstalled,
+// which would cause the namespace to get stuck in Terminating state and block the next CI run's install.
+var _ = ginkgo.SynchronizedAfterSuite(func() {}, func() {
+	cs, err := f.LoadClientset()
+	f.ExpectNoError(err, "creating kubernetes client")
+
+	ctx := context.Background()
+	f.Logf("Waiting for Mountpoint pods in mount-s3 namespace to be cleaned up...")
+	gomega.Eventually(ctx, func(ctx context.Context) (int, error) {
+		pods, err := cs.CoreV1().Pods("mount-s3").List(ctx, metav1.ListOptions{})
+		if err != nil {
+			return 0, err
+		}
+		if len(pods.Items) > 0 {
+			names := make([]string, len(pods.Items))
+			for i, pod := range pods.Items {
+				names[i] = pod.Name
+			}
+			f.Logf("Still waiting for %d Mountpoint pod(s) to be cleaned up: %v", len(pods.Items), names)
+		}
+		return len(pods.Items), nil
+	}).WithTimeout(5*time.Minute).WithPolling(10*time.Second).Should(gomega.Equal(0),
+		"Mountpoint pods in mount-s3 namespace were not cleaned up in time")
+	f.Logf("All Mountpoint pods cleaned up successfully")
+})
+
 // This executes testSuites for csi volumes.
 var _ = utils.SIGDescribe("CSI Volumes", func() {
 	var testSuites []func() framework.TestSuite